From 044e1ec2a815dd50c56c45fc3ea9eda5930bdd27 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 09:55:48 -0500 Subject: [PATCH 001/238] Add requirements file for linux container on Apple silicon --- requirements-linux-arm64.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 requirements-linux-arm64.txt diff --git a/requirements-linux-arm64.txt b/requirements-linux-arm64.txt new file mode 100644 index 0000000000..24ad623d84 --- /dev/null +++ b/requirements-linux-arm64.txt @@ -0,0 +1,24 @@ +albumentations==0.4.3 +einops==0.3.0 +huggingface-hub==0.8.1 +imageio==2.9.0 +imageio-ffmpeg==0.4.2 +kornia==0.6.0 +numpy==1.23.1 +--pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu +omegaconf==2.1.1 +opencv-python==4.6.0.66 +pillow==9.2.0 +pudb==2019.2 +torch==1.12.1 +torchvision==0.12.0 +pytorch-lightning==1.4.2 +streamlit==1.12.0 +test-tube>=0.7.5 +torch-fidelity==0.3.0 +torchmetrics==0.6.0 +transformers==4.19.2 +-e git+https://github.com/openai/CLIP.git@main#egg=clip +-e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers +git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion +-e . \ No newline at end of file From 348b4b8be5018432eec75853cefa31abf5f188bb Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 09:55:48 -0500 Subject: [PATCH 002/238] Add requirements file for linux container on Apple silicon --- requirements-linux-arm64.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 requirements-linux-arm64.txt diff --git a/requirements-linux-arm64.txt b/requirements-linux-arm64.txt new file mode 100644 index 0000000000..24ad623d84 --- /dev/null +++ b/requirements-linux-arm64.txt @@ -0,0 +1,24 @@ +albumentations==0.4.3 +einops==0.3.0 +huggingface-hub==0.8.1 +imageio==2.9.0 +imageio-ffmpeg==0.4.2 +kornia==0.6.0 +numpy==1.23.1 +--pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu +omegaconf==2.1.1 +opencv-python==4.6.0.66 +pillow==9.2.0 +pudb==2019.2 +torch==1.12.1 +torchvision==0.12.0 +pytorch-lightning==1.4.2 +streamlit==1.12.0 +test-tube>=0.7.5 +torch-fidelity==0.3.0 +torchmetrics==0.6.0 +transformers==4.19.2 +-e git+https://github.com/openai/CLIP.git@main#egg=clip +-e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers +git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion +-e . \ No newline at end of file From 4a0354c60443b5f9083516207fc995b5b79d6b03 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:28:15 -0500 Subject: [PATCH 003/238] Add README for Mac-Docker --- README-Mac-Docker.md | 234 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 README-Mac-Docker.md diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md new file mode 100644 index 0000000000..dc51944a37 --- /dev/null +++ b/README-Mac-Docker.md @@ -0,0 +1,234 @@ +Table of Contents +================= + +Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. + +* [Setup](#setup) + * [Directly on Apple silicon](#directly-on-apple-silicon) + * [Prerequisites](#prerequisites) + * [Set up](#set-up) + * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Prerequisites](#prerequisites-1) + * [Launch and set up a container](#launch-and-set-up-a-container) + * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) +* [Usage](#usage) + * [Startup](#startup) + * [Text to Image](#text-to-image) + * [Image to Image](#image-to-image) + * [Web Interface](#web-interface) + * [Notes](#notes) + +# Setup + +## Directly on Apple silicon + +For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). + +### Prerequisites +Install the latest versions of macOS, [Homebrew](https://brew.sh/), [Python](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#python), and [Git](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#git). + +```Shell +brew install cmake protobuf rust +brew install --cask miniconda +conda init zsh && source ~/.zshrc # or bash and .bashrc +``` + +### Set up +```Shell +GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +mkdir -p models/ldm/stable-diffusion-v1/ +``` + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: + +```Shell +PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt +ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 +conda env create -f environment-mac.yaml && conda activate ldm +``` + +You can verify you're in the virtual environment by looking at which executable you're getting: +```Shell +type python3 +``` + +Only need to do this once: +```Shell +python3 scripts/preload_models.py +``` + +## On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --name dummy --mount source=my-vol,target=/data alpine +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Launch and set up a container +Start a container for Stable Diffusion +```Shell +docker run -it \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +debian +# or arm64v8/debian +``` + +You're now on the container. Set it up: +```Shell +apt update && apt upgrade -y && apt install -y \ +git \ +pip3 \ +python3 \ +wget + +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + +# you won't need to close and reopen your terminal after this because we'll source our .rc file +cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc +# update conda +conda update -y -n base -c defaults conda + +cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 + +# Create the environment +# conda env create -f environment.yaml && conda activate ldm +conda create -y --name ldm && conda activate ldm +pip3 install -r requirements-linux-arm64.txt + +python3 scripts/preload_models.py + +mkdir -p models/ldm/stable-diffusion-v1 \ +&& chown root:root /data/sd-v1-4.ckpt \ +&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt +``` + +## [Optional] Face Restoration and Upscaling +```Shell +cd .. # by default expected in a sibling directory +git clone https://github.com/TencentARC/GFPGAN.git +cd GFPGAN + +pip3 install basicsr # used for training and inference +pip3 install facexlib # face detection and face restoration helper +pip3 install -r requirements.txt + +python3 setup.py develop +pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling +# pre-trained model needed for face restoration +wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + +cd .. +cd stable-diffusion +python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +``` + +# Usage + +## Startup +With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: +Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. + +By default the images are saved in ```outputs/img-samples/```. +If you're on a docker container set the output dir to the Docker volume. +```Shell +# If on Macbook +python3 scripts/dream.py --full_precision +# If on Linux container +python3 scripts/dream.py --full_precision -o /data +``` + +You'll get the script's prompt. You can see available options or quit. +```Shell +dream> -h +dream> q +``` + +## Text to Image +For quick (and rough) results test with 5 steps (default 50), 1 sample image. +Increase steps to 100 or more for good (but slower) results. +The prompt can be in quotes or not. +``` +dream> The hulk fighting with sheldon cooper -s5 -n1 +dream> "woman closeup highly detailed" -s 150 +# Reuse previous seed and apply face restoration (if you installed GFPGAN) +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +``` +TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. + +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +```Shell +# On your host Macbook (you can use the name of any container that mounted the volume) +docker cp dummy:/data/ ~/Pictures +``` + +## Image to Image +You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing. +Strength is a value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely. +Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256. + +If you're on a docker container, copy your input image into the Docker volume +```Shell +docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +``` + +Try it out generating an image (or 4). +```Shell +# If you're on your Macbook +dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +# If you're on a Linux container on your Macbook +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +``` + +## Web Interface +You can use the script with a graphical web interface +```Shell +python3 scripts/dream.py --full_precision --web +``` +and point your browser to http://127.0.0.1:9090 + +## Notes + +Some text you can add at the end of the prompt to make it very pretty: +```Shell +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +``` + +The original scripts should work as well. +```Shell +python3 scripts/orig_scripts/txt2img.py --help +python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1 --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray" +python3 scripts/orig_scripts/txt2img.py --ddim_steps 5 --n_iter 1 --n_samples 1 --plms --prompt "ocean" # or --klms +``` + From 6d2084e0303c88e8ebe031468ca870dc69f1c4a6 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:46:14 -0500 Subject: [PATCH 004/238] Clean up instructioins --- README-Mac-Docker.md | 51 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index dc51944a37..ae51416e6f 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -1,16 +1,16 @@ + Table of Contents ================= -Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. - -* [Setup](#setup) - * [Directly on Apple silicon](#directly-on-apple-silicon) +* [Installation](#installation) + * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) * [Prerequisites](#prerequisites) - * [Set up](#set-up) - * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Setup](#setup) + * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites-1) - * [Launch and set up a container](#launch-and-set-up-a-container) + * [Setup](#setup-1) * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) + * [Setup](#setup-2) * [Usage](#usage) * [Startup](#startup) * [Text to Image](#text-to-image) @@ -18,10 +18,9 @@ Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. * [Web Interface](#web-interface) * [Notes](#notes) -# Setup +# Installation -## Directly on Apple silicon - +## Option 1 - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -33,7 +32,8 @@ brew install --cask miniconda conda init zsh && source ~/.zshrc # or bash and .bashrc ``` -### Set up +### Setup +Set it to the fork you want to use. ```Shell GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git @@ -65,7 +65,7 @@ Only need to do this once: python3 scripts/preload_models.py ``` -## On a Linux container with Docker for Apple silicon +## Option 2 - On a Linux container with Docker for Apple silicon You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. ### Prerequisites @@ -87,7 +87,7 @@ cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` -### Launch and set up a container +### Setup Start a container for Stable Diffusion ```Shell docker run -it \ @@ -99,16 +99,16 @@ debian # or arm64v8/debian ``` -You're now on the container. Set it up: +You're now on the container. Set the fork you want to use and set up the container: ```Shell +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + apt update && apt upgrade -y && apt install -y \ git \ pip3 \ python3 \ wget -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc @@ -135,13 +135,15 @@ mkdir -p models/ldm/stable-diffusion-v1 \ ``` ## [Optional] Face Restoration and Upscaling -```Shell -cd .. # by default expected in a sibling directory -git clone https://github.com/TencentARC/GFPGAN.git -cd GFPGAN +Whether you're directly on macOS or a Linux container. -pip3 install basicsr # used for training and inference -pip3 install facexlib # face detection and face restoration helper +### Setup +```Shell +# by default expected in a sibling directory to stable-diffusion +cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN + +# basicsr: used for training and inference. facexlib: face detection / face restoration helper. +pip3 install basicsr facexlib pip3 install -r requirements.txt python3 setup.py develop @@ -149,8 +151,7 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up # pre-trained model needed for face restoration wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models -cd .. -cd stable-diffusion +cd ../stable-diffusion python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. ``` @@ -222,7 +223,7 @@ and point your browser to http://127.0.0.1:9090 Some text you can add at the end of the prompt to make it very pretty: ```Shell -cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed, painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida ``` The original scripts should work as well. From 4185afea5c299b146123a49abde1d6d2fac91ba4 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 23:15:09 -0500 Subject: [PATCH 005/238] Update documentation --- README-Mac-Docker.md | 85 +++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index ae51416e6f..503fe7f442 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -18,6 +18,10 @@ Table of Contents * [Web Interface](#web-interface) * [Notes](#notes) + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + # Installation ## Option 1 - Directly on Apple silicon @@ -33,8 +37,9 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ``` ### Setup -Set it to the fork you want to use. + ```Shell +# Set the fork you want to use. GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git git clone $GITHUB_STABLE_DIFFUSION @@ -42,8 +47,6 @@ cd stable-diffusion mkdir -p models/ldm/stable-diffusion-v1/ ``` -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: - ```Shell PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt @@ -70,10 +73,7 @@ You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https: ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. -You'll need to create an account but it's quick and free. +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. Create a Docker volume for the downloaded model file ``` @@ -82,38 +82,39 @@ docker volume create my-vol Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. ```Shell -docker create --name dummy --mount source=my-vol,target=/data alpine +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` ### Setup -Start a container for Stable Diffusion +Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. ```Shell docker run -it \ --platform linux/arm64 \ --name stable-diffusion \ --hostname stable-diffusion \ --mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 80:9090 \ debian # or arm64v8/debian ``` -You're now on the container. Set the fork you want to use and set up the container: +You're now on the container. ```Shell -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - -apt update && apt upgrade -y && apt install -y \ +# Set the fork you want to use +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ +&& apt update && apt upgrade -y \ +&& apt install -y \ git \ -pip3 \ +pip \ python3 \ wget # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc -# update conda -conda update -y -n base -c defaults conda cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion @@ -122,11 +123,11 @@ cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 -# Create the environment -# conda env create -f environment.yaml && conda activate ldm -conda create -y --name ldm && conda activate ldm -pip3 install -r requirements-linux-arm64.txt +# Create the environment, activate it, install requirements. +conda create -y --name ldm && conda activate ldm \ +&& pip3 install -r requirements-linux-arm64.txt +# Only need to do this once (ok twice if you decide to add face restoration and upscaling): python3 scripts/preload_models.py mkdir -p models/ldm/stable-diffusion-v1 \ @@ -139,6 +140,9 @@ Whether you're directly on macOS or a Linux container. ### Setup ```Shell +# If you're on a Linux container +apt install -y libgl1-mesa-glx libglib2.0-0 + # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN @@ -152,9 +156,11 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +python3 scripts/preload_models.py ``` + # Usage ## Startup @@ -162,12 +168,12 @@ With the Conda environment activated (```conda activate ldm```), run the interac Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume. +If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision # If on Linux container -python3 scripts/dream.py --full_precision -o /data +python3 scripts/dream.py --full_precision -o /data ``` You'll get the script's prompt. You can see available options or quit. @@ -177,21 +183,25 @@ dream> q ``` ## Text to Image -For quick (and rough) results test with 5 steps (default 50), 1 sample image. -Increase steps to 100 or more for good (but slower) results. +For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. -``` +```Shell dream> The hulk fighting with sheldon cooper -s5 -n1 dream> "woman closeup highly detailed" -s 150 # Reuse previous seed and apply face restoration (if you installed GFPGAN) -dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 +# TODO: example for upscaling. ``` -TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +You can download it from the Docker Desktop app, Volumes, my-vol, data. +Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) -docker cp dummy:/data/ ~/Pictures +docker cp dummy:/data/000001.928403745.png /Users//Pictures ``` ## Image to Image @@ -201,23 +211,26 @@ Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Oth If you're on a docker container, copy your input image into the Docker volume ```Shell -docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +docker cp /Users//Pictures/sketch-mountains-input.jpg dummy:/data/ ``` -Try it out generating an image (or 4). +Try it out generating an image (or 4). +The ```dream``` script needs absolute paths to find the image so don't use ```~```. ```Shell # If you're on your Macbook -dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +dream> "A fantasy landscape, trending on artstation" -I /Users//Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 # If you're on a Linux container on your Macbook -dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n1 ``` ## Web Interface -You can use the script with a graphical web interface +You can use the ```dream``` script with a graphical web interface. Start the web server with: ```Shell python3 scripts/dream.py --full_precision --web ``` -and point your browser to http://127.0.0.1:9090 +If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090 + +Press Control-C at the command line to stop the web server. ## Notes From dbfc35ece21e718f9318670686dcb7a50e816935 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:28:15 -0500 Subject: [PATCH 006/238] Add README for Mac-Docker --- README-Mac-Docker.md | 234 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 README-Mac-Docker.md diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md new file mode 100644 index 0000000000..dc51944a37 --- /dev/null +++ b/README-Mac-Docker.md @@ -0,0 +1,234 @@ +Table of Contents +================= + +Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. + +* [Setup](#setup) + * [Directly on Apple silicon](#directly-on-apple-silicon) + * [Prerequisites](#prerequisites) + * [Set up](#set-up) + * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Prerequisites](#prerequisites-1) + * [Launch and set up a container](#launch-and-set-up-a-container) + * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) +* [Usage](#usage) + * [Startup](#startup) + * [Text to Image](#text-to-image) + * [Image to Image](#image-to-image) + * [Web Interface](#web-interface) + * [Notes](#notes) + +# Setup + +## Directly on Apple silicon + +For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). + +### Prerequisites +Install the latest versions of macOS, [Homebrew](https://brew.sh/), [Python](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#python), and [Git](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#git). + +```Shell +brew install cmake protobuf rust +brew install --cask miniconda +conda init zsh && source ~/.zshrc # or bash and .bashrc +``` + +### Set up +```Shell +GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +mkdir -p models/ldm/stable-diffusion-v1/ +``` + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: + +```Shell +PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt +ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 +conda env create -f environment-mac.yaml && conda activate ldm +``` + +You can verify you're in the virtual environment by looking at which executable you're getting: +```Shell +type python3 +``` + +Only need to do this once: +```Shell +python3 scripts/preload_models.py +``` + +## On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --name dummy --mount source=my-vol,target=/data alpine +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Launch and set up a container +Start a container for Stable Diffusion +```Shell +docker run -it \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +debian +# or arm64v8/debian +``` + +You're now on the container. Set it up: +```Shell +apt update && apt upgrade -y && apt install -y \ +git \ +pip3 \ +python3 \ +wget + +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + +# you won't need to close and reopen your terminal after this because we'll source our .rc file +cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc +# update conda +conda update -y -n base -c defaults conda + +cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 + +# Create the environment +# conda env create -f environment.yaml && conda activate ldm +conda create -y --name ldm && conda activate ldm +pip3 install -r requirements-linux-arm64.txt + +python3 scripts/preload_models.py + +mkdir -p models/ldm/stable-diffusion-v1 \ +&& chown root:root /data/sd-v1-4.ckpt \ +&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt +``` + +## [Optional] Face Restoration and Upscaling +```Shell +cd .. # by default expected in a sibling directory +git clone https://github.com/TencentARC/GFPGAN.git +cd GFPGAN + +pip3 install basicsr # used for training and inference +pip3 install facexlib # face detection and face restoration helper +pip3 install -r requirements.txt + +python3 setup.py develop +pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling +# pre-trained model needed for face restoration +wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + +cd .. +cd stable-diffusion +python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +``` + +# Usage + +## Startup +With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: +Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. + +By default the images are saved in ```outputs/img-samples/```. +If you're on a docker container set the output dir to the Docker volume. +```Shell +# If on Macbook +python3 scripts/dream.py --full_precision +# If on Linux container +python3 scripts/dream.py --full_precision -o /data +``` + +You'll get the script's prompt. You can see available options or quit. +```Shell +dream> -h +dream> q +``` + +## Text to Image +For quick (and rough) results test with 5 steps (default 50), 1 sample image. +Increase steps to 100 or more for good (but slower) results. +The prompt can be in quotes or not. +``` +dream> The hulk fighting with sheldon cooper -s5 -n1 +dream> "woman closeup highly detailed" -s 150 +# Reuse previous seed and apply face restoration (if you installed GFPGAN) +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +``` +TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. + +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +```Shell +# On your host Macbook (you can use the name of any container that mounted the volume) +docker cp dummy:/data/ ~/Pictures +``` + +## Image to Image +You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing. +Strength is a value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely. +Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256. + +If you're on a docker container, copy your input image into the Docker volume +```Shell +docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +``` + +Try it out generating an image (or 4). +```Shell +# If you're on your Macbook +dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +# If you're on a Linux container on your Macbook +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +``` + +## Web Interface +You can use the script with a graphical web interface +```Shell +python3 scripts/dream.py --full_precision --web +``` +and point your browser to http://127.0.0.1:9090 + +## Notes + +Some text you can add at the end of the prompt to make it very pretty: +```Shell +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +``` + +The original scripts should work as well. +```Shell +python3 scripts/orig_scripts/txt2img.py --help +python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1 --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray" +python3 scripts/orig_scripts/txt2img.py --ddim_steps 5 --n_iter 1 --n_samples 1 --plms --prompt "ocean" # or --klms +``` + From e3be28eccaab3cd315030e88aba05a589c9d41ff Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:46:14 -0500 Subject: [PATCH 007/238] Clean up instructioins --- README-Mac-Docker.md | 51 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index dc51944a37..ae51416e6f 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -1,16 +1,16 @@ + Table of Contents ================= -Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. - -* [Setup](#setup) - * [Directly on Apple silicon](#directly-on-apple-silicon) +* [Installation](#installation) + * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) * [Prerequisites](#prerequisites) - * [Set up](#set-up) - * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Setup](#setup) + * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites-1) - * [Launch and set up a container](#launch-and-set-up-a-container) + * [Setup](#setup-1) * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) + * [Setup](#setup-2) * [Usage](#usage) * [Startup](#startup) * [Text to Image](#text-to-image) @@ -18,10 +18,9 @@ Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. * [Web Interface](#web-interface) * [Notes](#notes) -# Setup +# Installation -## Directly on Apple silicon - +## Option 1 - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -33,7 +32,8 @@ brew install --cask miniconda conda init zsh && source ~/.zshrc # or bash and .bashrc ``` -### Set up +### Setup +Set it to the fork you want to use. ```Shell GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git @@ -65,7 +65,7 @@ Only need to do this once: python3 scripts/preload_models.py ``` -## On a Linux container with Docker for Apple silicon +## Option 2 - On a Linux container with Docker for Apple silicon You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. ### Prerequisites @@ -87,7 +87,7 @@ cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` -### Launch and set up a container +### Setup Start a container for Stable Diffusion ```Shell docker run -it \ @@ -99,16 +99,16 @@ debian # or arm64v8/debian ``` -You're now on the container. Set it up: +You're now on the container. Set the fork you want to use and set up the container: ```Shell +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + apt update && apt upgrade -y && apt install -y \ git \ pip3 \ python3 \ wget -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc @@ -135,13 +135,15 @@ mkdir -p models/ldm/stable-diffusion-v1 \ ``` ## [Optional] Face Restoration and Upscaling -```Shell -cd .. # by default expected in a sibling directory -git clone https://github.com/TencentARC/GFPGAN.git -cd GFPGAN +Whether you're directly on macOS or a Linux container. -pip3 install basicsr # used for training and inference -pip3 install facexlib # face detection and face restoration helper +### Setup +```Shell +# by default expected in a sibling directory to stable-diffusion +cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN + +# basicsr: used for training and inference. facexlib: face detection / face restoration helper. +pip3 install basicsr facexlib pip3 install -r requirements.txt python3 setup.py develop @@ -149,8 +151,7 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up # pre-trained model needed for face restoration wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models -cd .. -cd stable-diffusion +cd ../stable-diffusion python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. ``` @@ -222,7 +223,7 @@ and point your browser to http://127.0.0.1:9090 Some text you can add at the end of the prompt to make it very pretty: ```Shell -cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed, painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida ``` The original scripts should work as well. From f74e52079bc3c0baf82b36dfa2e420837f75f6bf Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 23:15:09 -0500 Subject: [PATCH 008/238] Update documentation --- README-Mac-Docker.md | 85 +++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index ae51416e6f..503fe7f442 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -18,6 +18,10 @@ Table of Contents * [Web Interface](#web-interface) * [Notes](#notes) + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + # Installation ## Option 1 - Directly on Apple silicon @@ -33,8 +37,9 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ``` ### Setup -Set it to the fork you want to use. + ```Shell +# Set the fork you want to use. GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git git clone $GITHUB_STABLE_DIFFUSION @@ -42,8 +47,6 @@ cd stable-diffusion mkdir -p models/ldm/stable-diffusion-v1/ ``` -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: - ```Shell PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt @@ -70,10 +73,7 @@ You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https: ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. -You'll need to create an account but it's quick and free. +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. Create a Docker volume for the downloaded model file ``` @@ -82,38 +82,39 @@ docker volume create my-vol Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. ```Shell -docker create --name dummy --mount source=my-vol,target=/data alpine +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` ### Setup -Start a container for Stable Diffusion +Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. ```Shell docker run -it \ --platform linux/arm64 \ --name stable-diffusion \ --hostname stable-diffusion \ --mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 80:9090 \ debian # or arm64v8/debian ``` -You're now on the container. Set the fork you want to use and set up the container: +You're now on the container. ```Shell -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - -apt update && apt upgrade -y && apt install -y \ +# Set the fork you want to use +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ +&& apt update && apt upgrade -y \ +&& apt install -y \ git \ -pip3 \ +pip \ python3 \ wget # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc -# update conda -conda update -y -n base -c defaults conda cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion @@ -122,11 +123,11 @@ cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 -# Create the environment -# conda env create -f environment.yaml && conda activate ldm -conda create -y --name ldm && conda activate ldm -pip3 install -r requirements-linux-arm64.txt +# Create the environment, activate it, install requirements. +conda create -y --name ldm && conda activate ldm \ +&& pip3 install -r requirements-linux-arm64.txt +# Only need to do this once (ok twice if you decide to add face restoration and upscaling): python3 scripts/preload_models.py mkdir -p models/ldm/stable-diffusion-v1 \ @@ -139,6 +140,9 @@ Whether you're directly on macOS or a Linux container. ### Setup ```Shell +# If you're on a Linux container +apt install -y libgl1-mesa-glx libglib2.0-0 + # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN @@ -152,9 +156,11 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +python3 scripts/preload_models.py ``` + # Usage ## Startup @@ -162,12 +168,12 @@ With the Conda environment activated (```conda activate ldm```), run the interac Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume. +If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision # If on Linux container -python3 scripts/dream.py --full_precision -o /data +python3 scripts/dream.py --full_precision -o /data ``` You'll get the script's prompt. You can see available options or quit. @@ -177,21 +183,25 @@ dream> q ``` ## Text to Image -For quick (and rough) results test with 5 steps (default 50), 1 sample image. -Increase steps to 100 or more for good (but slower) results. +For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. -``` +```Shell dream> The hulk fighting with sheldon cooper -s5 -n1 dream> "woman closeup highly detailed" -s 150 # Reuse previous seed and apply face restoration (if you installed GFPGAN) -dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 +# TODO: example for upscaling. ``` -TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +You can download it from the Docker Desktop app, Volumes, my-vol, data. +Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) -docker cp dummy:/data/ ~/Pictures +docker cp dummy:/data/000001.928403745.png /Users//Pictures ``` ## Image to Image @@ -201,23 +211,26 @@ Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Oth If you're on a docker container, copy your input image into the Docker volume ```Shell -docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +docker cp /Users//Pictures/sketch-mountains-input.jpg dummy:/data/ ``` -Try it out generating an image (or 4). +Try it out generating an image (or 4). +The ```dream``` script needs absolute paths to find the image so don't use ```~```. ```Shell # If you're on your Macbook -dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +dream> "A fantasy landscape, trending on artstation" -I /Users//Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 # If you're on a Linux container on your Macbook -dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n1 ``` ## Web Interface -You can use the script with a graphical web interface +You can use the ```dream``` script with a graphical web interface. Start the web server with: ```Shell python3 scripts/dream.py --full_precision --web ``` -and point your browser to http://127.0.0.1:9090 +If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090 + +Press Control-C at the command line to stop the web server. ## Notes From 97c0c4bfe817d98e6031030450e7fac0b0bb3af7 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:38:31 -0500 Subject: [PATCH 009/238] Initial version of Dockerfile and entreypoint --- Dockerfile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ entrypoint.sh | 8 ++++++++ 2 files changed, 58 insertions(+) create mode 100644 Dockerfile create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..59382247fb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +FROM arm64v8/debian +MAINTAINER Armando C. Santisbon + +ARG gsd +ENV GITHUB_STABLE_DIFFUSION $gsd + +ARG sdreq="requirements-linux-arm64.txt" +ENV SD_REQ $sdreq + +WORKDIR / +COPY entrypoint.sh anaconda.sh . +SHELL ["/bin/bash", "-c"] + +RUN apt update && apt upgrade -y \ + && apt install -y \ + git \ + pip \ + python3 \ + wget \ + # install Anaconda or Miniconda + && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ + && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion \ + # When path exists, pip3 will (w)ipe. + && PIP_EXISTS_ACTION="w" \ + # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. + && CONDA_SUBDIR="osx-arm64" \ + # Create the environment, activate it, install requirements. + && conda create -y --name ldm && conda activate ldm \ + && pip3 install -r $SD_REQ \ + + # Only need to do this once (we'll do it after we add face restoration and upscaling): + # && python3 scripts/preload_models.py \ + + && mkdir models/ldm/stable-diffusion-v1 \ + # [Optional] Face Restoration and Upscaling + && apt install -y libgl1-mesa-glx libglib2.0-0 \ + # by default expected in a sibling directory to stable-diffusion + && cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN \ + && pip3 install basicsr facexlib \ + && pip3 install -r requirements.txt \ + && python3 setup.py develop \ + # to enhance the background (non-face) regions and do upscaling + && pip3 install realesrgan \ + # pre-trained model needed for face restoration + && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models \ + && cd ../stable-diffusion \ + # if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. + && python3 scripts/preload_models.py + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000000..26967ab8ae --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt + +if [ $# -eq 0 ]; then + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data +else + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" +fi From f9239af7dc6f1c57bc51ad26ca8321cc989822e3 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:42:05 -0500 Subject: [PATCH 010/238] Ignore Anaconda/Miniconda installer used during Docker build --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fd75e65a48..bbb6fc20ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ outputs/ models/ldm/stable-diffusion-v1/model.ckpt +# ignore the Anaconda/Miniconda installer used while building Docker image +anaconda.sh + # ignore a directory which serves as a place for initial images inputs/ From 2cf8de92344bb63ac5e7a3487a65f86b5ccbd725 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 13:56:18 -0500 Subject: [PATCH 011/238] Add entrypoint and update documentation --- README-Mac-Docker.md | 186 +++++++++++++++++++------------------------ entrypoint.sh | 6 +- 2 files changed, 87 insertions(+), 105 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index 503fe7f442..eda202152c 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -2,29 +2,84 @@ Table of Contents ================= -* [Installation](#installation) - * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) +* [Step 1 - Get the Model](#step-1---get-the-model) +* [Step 2 - Installation](#step-2---installation) + * [Option A - On a Linux container with Docker for Apple silicon](#option-a---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites) * [Setup](#setup) - * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) + * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) * [Prerequisites](#prerequisites-1) * [Setup](#setup-1) - * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) - * [Setup](#setup-2) -* [Usage](#usage) +* [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) * [Startup](#startup) * [Text to Image](#text-to-image) * [Image to Image](#image-to-image) * [Web Interface](#web-interface) * [Notes](#notes) - +# Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. -# Installation +# Step 2 - Installation -## Option 1 - Directly on Apple silicon +## Option A - On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine + +# Copy the model file to the Docker volume. We'll need it at run time. +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Setup +```Shell +# Set the fork you want to use. +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +chmod +x entrypoint.sh +# download the Miniconda installer. We'll need it at build time. +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +``` + +Build the Docker image. Give it any tag ```-t``` that you want. +Tip: Make sure your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. +```Shell +docker build -t santisbon/stable-diffusion \ +--build-arg gsd=$GITHUB_STABLE_DIFFUSION \ +--build-arg sdreq="requirements-linux-arm64.txt" \ +. +``` + +Run a container using your built image e.g. +```Shell +docker run -it \ +--rm \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 9090:9090 \ +santisbon/stable-diffusion +``` +Tip: Make sure you've created the Docker volume (above) + +## Option B - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -40,7 +95,7 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ```Shell # Set the fork you want to use. -GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion @@ -53,8 +108,8 @@ ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt # When path exists, pip3 will (w)ipe. # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 +PIP_EXISTS_ACTION="w" +CONDA_SUBDIR="osx-arm64" conda env create -f environment-mac.yaml && conda activate ldm ``` @@ -63,92 +118,15 @@ You can verify you're in the virtual environment by looking at which executable type python3 ``` -Only need to do this once: +Face Restoration and Upscaling ```Shell -python3 scripts/preload_models.py -``` - -## Option 2 - On a Linux container with Docker for Apple silicon -You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. - -### Prerequisites -[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Create a Docker volume for the downloaded model file -``` -docker volume create my-vol -``` - -Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. -```Shell -docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine -cd ~/Downloads # or wherever you saved sd-v1-4.ckpt -docker cp sd-v1-4.ckpt dummy:/data -``` - -### Setup -Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. -```Shell -docker run -it \ ---platform linux/arm64 \ ---name stable-diffusion \ ---hostname stable-diffusion \ ---mount source=my-vol,target=/data \ ---expose 9090 \ ---publish 80:9090 \ -debian -# or arm64v8/debian -``` - -You're now on the container. -```Shell -# Set the fork you want to use -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ -&& apt update && apt upgrade -y \ -&& apt install -y \ -git \ -pip \ -python3 \ -wget - -# you won't need to close and reopen your terminal after this because we'll source our .rc file -cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ -&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc - -cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion - -# When path exists, pip3 will (w)ipe. -# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 - -# Create the environment, activate it, install requirements. -conda create -y --name ldm && conda activate ldm \ -&& pip3 install -r requirements-linux-arm64.txt - -# Only need to do this once (ok twice if you decide to add face restoration and upscaling): -python3 scripts/preload_models.py - -mkdir -p models/ldm/stable-diffusion-v1 \ -&& chown root:root /data/sd-v1-4.ckpt \ -&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt -``` - -## [Optional] Face Restoration and Upscaling -Whether you're directly on macOS or a Linux container. - -### Setup -```Shell -# If you're on a Linux container -apt install -y libgl1-mesa-glx libglib2.0-0 # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN # basicsr: used for training and inference. facexlib: face detection / face restoration helper. -pip3 install basicsr facexlib -pip3 install -r requirements.txt +pip3 install basicsr facexlib \ +&& pip3 install -r requirements.txt python3 setup.py develop pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling @@ -156,24 +134,26 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. -python3 scripts/preload_models.py ``` +Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. +```Shell +python3 scripts/preload_models.py +``` -# Usage +# Step 3 - Usage (time to have fun) ## Startup +If you're on a Linux container the ```dream``` script is automatically started and the output dir set to the Docker volume you created earlier. + +If you're directly on macOS follow these startup instructions. With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. -By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision -# If on Linux container -python3 scripts/dream.py --full_precision -o /data +# By default the images are saved in outputs/img-samples/. ``` You'll get the script's prompt. You can see available options or quit. @@ -183,7 +163,7 @@ dream> q ``` ## Text to Image -For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +For quick (but bad) image results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. ```Shell @@ -194,11 +174,11 @@ dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 # TODO: example for upscaling. ``` You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. -The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +If you're on a container the output is set to the Docker volume. You can copy it wherever you want. You can download it from the Docker Desktop app, Volumes, my-vol, data. -Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: +Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) docker cp dummy:/data/000001.928403745.png /Users//Pictures diff --git a/entrypoint.sh b/entrypoint.sh index 26967ab8ae..c7cc9af8d3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,8 +1,10 @@ #!/bin/bash ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt +cd /stable-diffusion +conda activate ldm if [ $# -eq 0 ]; then - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data + python3 scripts/dream.py --full_precision -o /data else - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" + python3 scripts/dream.py --full_precision -o /data "$@" fi From 5cbea51f3185cc42db9708b2ab7a88191695bcb8 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:38:31 -0500 Subject: [PATCH 012/238] Initial version of Dockerfile and entreypoint --- Dockerfile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ entrypoint.sh | 8 ++++++++ 2 files changed, 58 insertions(+) create mode 100644 Dockerfile create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..59382247fb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +FROM arm64v8/debian +MAINTAINER Armando C. Santisbon + +ARG gsd +ENV GITHUB_STABLE_DIFFUSION $gsd + +ARG sdreq="requirements-linux-arm64.txt" +ENV SD_REQ $sdreq + +WORKDIR / +COPY entrypoint.sh anaconda.sh . +SHELL ["/bin/bash", "-c"] + +RUN apt update && apt upgrade -y \ + && apt install -y \ + git \ + pip \ + python3 \ + wget \ + # install Anaconda or Miniconda + && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ + && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion \ + # When path exists, pip3 will (w)ipe. + && PIP_EXISTS_ACTION="w" \ + # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. + && CONDA_SUBDIR="osx-arm64" \ + # Create the environment, activate it, install requirements. + && conda create -y --name ldm && conda activate ldm \ + && pip3 install -r $SD_REQ \ + + # Only need to do this once (we'll do it after we add face restoration and upscaling): + # && python3 scripts/preload_models.py \ + + && mkdir models/ldm/stable-diffusion-v1 \ + # [Optional] Face Restoration and Upscaling + && apt install -y libgl1-mesa-glx libglib2.0-0 \ + # by default expected in a sibling directory to stable-diffusion + && cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN \ + && pip3 install basicsr facexlib \ + && pip3 install -r requirements.txt \ + && python3 setup.py develop \ + # to enhance the background (non-face) regions and do upscaling + && pip3 install realesrgan \ + # pre-trained model needed for face restoration + && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models \ + && cd ../stable-diffusion \ + # if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. + && python3 scripts/preload_models.py + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000000..26967ab8ae --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt + +if [ $# -eq 0 ]; then + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data +else + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" +fi From 7136603604ed02c4732d5514004a5f0be70f3ab5 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:42:05 -0500 Subject: [PATCH 013/238] Ignore Anaconda/Miniconda installer used during Docker build --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fd75e65a48..bbb6fc20ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ outputs/ models/ldm/stable-diffusion-v1/model.ckpt +# ignore the Anaconda/Miniconda installer used while building Docker image +anaconda.sh + # ignore a directory which serves as a place for initial images inputs/ From b656d333de9019739daaeb85b3a14172513fcae8 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 13:56:18 -0500 Subject: [PATCH 014/238] Add entrypoint and update documentation --- README-Mac-Docker.md | 186 +++++++++++++++++++------------------------ entrypoint.sh | 6 +- 2 files changed, 87 insertions(+), 105 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index 503fe7f442..eda202152c 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -2,29 +2,84 @@ Table of Contents ================= -* [Installation](#installation) - * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) +* [Step 1 - Get the Model](#step-1---get-the-model) +* [Step 2 - Installation](#step-2---installation) + * [Option A - On a Linux container with Docker for Apple silicon](#option-a---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites) * [Setup](#setup) - * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) + * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) * [Prerequisites](#prerequisites-1) * [Setup](#setup-1) - * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) - * [Setup](#setup-2) -* [Usage](#usage) +* [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) * [Startup](#startup) * [Text to Image](#text-to-image) * [Image to Image](#image-to-image) * [Web Interface](#web-interface) * [Notes](#notes) - +# Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. -# Installation +# Step 2 - Installation -## Option 1 - Directly on Apple silicon +## Option A - On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine + +# Copy the model file to the Docker volume. We'll need it at run time. +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Setup +```Shell +# Set the fork you want to use. +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +chmod +x entrypoint.sh +# download the Miniconda installer. We'll need it at build time. +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +``` + +Build the Docker image. Give it any tag ```-t``` that you want. +Tip: Make sure your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. +```Shell +docker build -t santisbon/stable-diffusion \ +--build-arg gsd=$GITHUB_STABLE_DIFFUSION \ +--build-arg sdreq="requirements-linux-arm64.txt" \ +. +``` + +Run a container using your built image e.g. +```Shell +docker run -it \ +--rm \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 9090:9090 \ +santisbon/stable-diffusion +``` +Tip: Make sure you've created the Docker volume (above) + +## Option B - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -40,7 +95,7 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ```Shell # Set the fork you want to use. -GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion @@ -53,8 +108,8 @@ ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt # When path exists, pip3 will (w)ipe. # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 +PIP_EXISTS_ACTION="w" +CONDA_SUBDIR="osx-arm64" conda env create -f environment-mac.yaml && conda activate ldm ``` @@ -63,92 +118,15 @@ You can verify you're in the virtual environment by looking at which executable type python3 ``` -Only need to do this once: +Face Restoration and Upscaling ```Shell -python3 scripts/preload_models.py -``` - -## Option 2 - On a Linux container with Docker for Apple silicon -You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. - -### Prerequisites -[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Create a Docker volume for the downloaded model file -``` -docker volume create my-vol -``` - -Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. -```Shell -docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine -cd ~/Downloads # or wherever you saved sd-v1-4.ckpt -docker cp sd-v1-4.ckpt dummy:/data -``` - -### Setup -Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. -```Shell -docker run -it \ ---platform linux/arm64 \ ---name stable-diffusion \ ---hostname stable-diffusion \ ---mount source=my-vol,target=/data \ ---expose 9090 \ ---publish 80:9090 \ -debian -# or arm64v8/debian -``` - -You're now on the container. -```Shell -# Set the fork you want to use -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ -&& apt update && apt upgrade -y \ -&& apt install -y \ -git \ -pip \ -python3 \ -wget - -# you won't need to close and reopen your terminal after this because we'll source our .rc file -cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ -&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc - -cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion - -# When path exists, pip3 will (w)ipe. -# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 - -# Create the environment, activate it, install requirements. -conda create -y --name ldm && conda activate ldm \ -&& pip3 install -r requirements-linux-arm64.txt - -# Only need to do this once (ok twice if you decide to add face restoration and upscaling): -python3 scripts/preload_models.py - -mkdir -p models/ldm/stable-diffusion-v1 \ -&& chown root:root /data/sd-v1-4.ckpt \ -&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt -``` - -## [Optional] Face Restoration and Upscaling -Whether you're directly on macOS or a Linux container. - -### Setup -```Shell -# If you're on a Linux container -apt install -y libgl1-mesa-glx libglib2.0-0 # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN # basicsr: used for training and inference. facexlib: face detection / face restoration helper. -pip3 install basicsr facexlib -pip3 install -r requirements.txt +pip3 install basicsr facexlib \ +&& pip3 install -r requirements.txt python3 setup.py develop pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling @@ -156,24 +134,26 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. -python3 scripts/preload_models.py ``` +Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. +```Shell +python3 scripts/preload_models.py +``` -# Usage +# Step 3 - Usage (time to have fun) ## Startup +If you're on a Linux container the ```dream``` script is automatically started and the output dir set to the Docker volume you created earlier. + +If you're directly on macOS follow these startup instructions. With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. -By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision -# If on Linux container -python3 scripts/dream.py --full_precision -o /data +# By default the images are saved in outputs/img-samples/. ``` You'll get the script's prompt. You can see available options or quit. @@ -183,7 +163,7 @@ dream> q ``` ## Text to Image -For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +For quick (but bad) image results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. ```Shell @@ -194,11 +174,11 @@ dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 # TODO: example for upscaling. ``` You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. -The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +If you're on a container the output is set to the Docker volume. You can copy it wherever you want. You can download it from the Docker Desktop app, Volumes, my-vol, data. -Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: +Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) docker cp dummy:/data/000001.928403745.png /Users//Pictures diff --git a/entrypoint.sh b/entrypoint.sh index 26967ab8ae..c7cc9af8d3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,8 +1,10 @@ #!/bin/bash ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt +cd /stable-diffusion +conda activate ldm if [ $# -eq 0 ]; then - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data + python3 scripts/dream.py --full_precision -o /data else - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" + python3 scripts/dream.py --full_precision -o /data "$@" fi From 529fc57f2bd22b742011823887de47e27e0716af Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Sat, 10 Sep 2022 16:58:07 +0300 Subject: [PATCH 015/238] ~7% speedup (1.57 to 1.69it/s) from switch to += in ldm.modules.attention. (#482) Tested on 8GB eGPU nvidia setup so YMMV. 512x512 output, max VRAM stays same. --- ldm/modules/attention.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 817e9bcdc4..24aef29279 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -235,9 +235,9 @@ class BasicTransformerBlock(nn.Module): def _forward(self, x, context=None): x = x.contiguous() if x.device.type == 'mps' else x - x = self.attn1(self.norm1(x)) + x - x = self.attn2(self.norm2(x), context=context) + x - x = self.ff(self.norm3(x)) + x + x += self.attn1(self.norm1(x)) + x += self.attn2(self.norm2(x), context=context) + x += self.ff(self.norm3(x)) return x From 9775a3502c6a3c3312a22b252a47301037dc8308 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 15:29:44 -0500 Subject: [PATCH 016/238] Fix documentation link --- README-Mac-Docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index eda202152c..b8b1bfd24b 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -27,7 +27,7 @@ You'll need to create an account but it's quick and free. You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. ### Prerequisites -[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#docker) On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. Create a Docker volume for the downloaded model file From eceb7d2b54b53b3bd7b64a0c1d1815928a031e04 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" <3804335+santisbon@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:52:12 -0500 Subject: [PATCH 017/238] [Documentation] Clarify path to use --- README-Mac-Docker.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index b8b1bfd24b..0f53742b6c 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -49,6 +49,7 @@ docker cp sd-v1-4.ckpt dummy:/data # Set the fork you want to use. GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" +cd ~ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion chmod +x entrypoint.sh From 336e16ef85e8b8e1983634f2bb01f71f1925ceec Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" <3804335+santisbon@users.noreply.github.com> Date: Sat, 10 Sep 2022 19:31:39 -0500 Subject: [PATCH 018/238] [Documentation] Fix typo --- README-Mac-Docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index 0f53742b6c..7541360a7c 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -54,7 +54,7 @@ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion chmod +x entrypoint.sh # download the Miniconda installer. We'll need it at build time. -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh ``` Build the Docker image. Give it any tag ```-t``` that you want. From eeff8e903341c7e9fa9eba4d2e682050edcb34d8 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 22:45:08 -0500 Subject: [PATCH 019/238] [Documentation] Add +x permissions to miniconda installer --- README-Mac-Docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index 7541360a7c..be2e837c54 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -54,7 +54,7 @@ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion chmod +x entrypoint.sh # download the Miniconda installer. We'll need it at build time. -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh ``` Build the Docker image. Give it any tag ```-t``` that you want. From e21938c12d94f8288aba69df9982013e463af461 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 02:04:11 -0500 Subject: [PATCH 020/238] Remove unnecesary chmod from Dockerfile to avoid bloating --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 59382247fb..7258b4a620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN apt update && apt upgrade -y \ python3 \ wget \ # install Anaconda or Miniconda - && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ + && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion \ # When path exists, pip3 will (w)ipe. && PIP_EXISTS_ACTION="w" \ From da95729d900b7e8e62423361d6ac3d40c2716dfc Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 09:47:54 -0500 Subject: [PATCH 021/238] Refactor docker build and move docker assets to their own folder --- Dockerfile | 50 ---------------- README-Mac-Docker.md | 5 +- docker-build/Dockerfile | 63 +++++++++++++++++++++ entrypoint.sh => docker-build/entrypoint.sh | 0 4 files changed, 66 insertions(+), 52 deletions(-) delete mode 100644 Dockerfile create mode 100644 docker-build/Dockerfile rename entrypoint.sh => docker-build/entrypoint.sh (100%) diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7258b4a620..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM arm64v8/debian -MAINTAINER Armando C. Santisbon - -ARG gsd -ENV GITHUB_STABLE_DIFFUSION $gsd - -ARG sdreq="requirements-linux-arm64.txt" -ENV SD_REQ $sdreq - -WORKDIR / -COPY entrypoint.sh anaconda.sh . -SHELL ["/bin/bash", "-c"] - -RUN apt update && apt upgrade -y \ - && apt install -y \ - git \ - pip \ - python3 \ - wget \ - # install Anaconda or Miniconda - && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ - && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion \ - # When path exists, pip3 will (w)ipe. - && PIP_EXISTS_ACTION="w" \ - # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. - && CONDA_SUBDIR="osx-arm64" \ - # Create the environment, activate it, install requirements. - && conda create -y --name ldm && conda activate ldm \ - && pip3 install -r $SD_REQ \ - - # Only need to do this once (we'll do it after we add face restoration and upscaling): - # && python3 scripts/preload_models.py \ - - && mkdir models/ldm/stable-diffusion-v1 \ - # [Optional] Face Restoration and Upscaling - && apt install -y libgl1-mesa-glx libglib2.0-0 \ - # by default expected in a sibling directory to stable-diffusion - && cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN \ - && pip3 install basicsr facexlib \ - && pip3 install -r requirements.txt \ - && python3 setup.py develop \ - # to enhance the background (non-face) regions and do upscaling - && pip3 install realesrgan \ - # pre-trained model needed for face restoration - && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models \ - && cd ../stable-diffusion \ - # if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. - && python3 scripts/preload_models.py - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index be2e837c54..b2f9a93a8d 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -51,7 +51,8 @@ GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" cd ~ git clone $GITHUB_STABLE_DIFFUSION -cd stable-diffusion + +cd stable-diffusion/docker-build chmod +x entrypoint.sh # download the Miniconda installer. We'll need it at build time. wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh @@ -137,7 +138,7 @@ wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pt cd ../stable-diffusion ``` -Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. +Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. Used by the core functionality and by GFPGAN/Real-ESRGAN. ```Shell python3 scripts/preload_models.py ``` diff --git a/docker-build/Dockerfile b/docker-build/Dockerfile new file mode 100644 index 0000000000..3f4a8a0160 --- /dev/null +++ b/docker-build/Dockerfile @@ -0,0 +1,63 @@ +FROM arm64v8/debian + +ARG gsd +ENV GITHUB_STABLE_DIFFUSION $gsd +ARG sdreq="requirements-linux-arm64.txt" +ENV SD_REQ $sdreq + +WORKDIR / + +COPY entrypoint.sh anaconda.sh . +SHELL ["/bin/bash", "-c"] + +# Update and apt +RUN apt update && apt upgrade -y \ + && apt install -y \ + git \ + pip \ + python3 \ + wget + +# install Anaconda or Miniconda +RUN bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash + +# SD repo +RUN git clone $GITHUB_STABLE_DIFFUSION + +WORKDIR /stable-diffusion + +# SD env +RUN PIP_EXISTS_ACTION="w" \ + # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. + && CONDA_SUBDIR="osx-arm64" \ + # Create the environment, activate it, install requirements. + && source ~/.bashrc && conda create -y --name ldm && conda activate ldm \ + && pip3 install -r $SD_REQ \ + && mkdir models/ldm/stable-diffusion-v1 + +# Face restoration prerequisites +RUN apt install -y libgl1-mesa-glx libglib2.0-0 + # by default expected in a sibling directory to stable-diffusion + +WORKDIR / + +# Face restoreation repo +RUN git clone https://github.com/TencentARC/GFPGAN.git + +WORKDIR /GFPGAN + +# Face restoration env +RUN pip3 install basicsr facexlib \ + && pip3 install -r requirements.txt \ + && python3 setup.py develop \ + # to enhance the background (non-face) regions and do upscaling + && pip3 install realesrgan \ + # pre-trained model needed for face restoration + && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + +WORKDIR /stable-diffusion + +# Preload models +RUN python3 scripts/preload_models.py + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/entrypoint.sh b/docker-build/entrypoint.sh similarity index 100% rename from entrypoint.sh rename to docker-build/entrypoint.sh From 19fb66f3d55c2b4b672f148f61417ab0d4e42ee3 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 11:03:33 -0500 Subject: [PATCH 022/238] Add comments on next steps --- docker-build/Dockerfile | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docker-build/Dockerfile b/docker-build/Dockerfile index 3f4a8a0160..3f57b14e18 100644 --- a/docker-build/Dockerfile +++ b/docker-build/Dockerfile @@ -7,10 +7,12 @@ ENV SD_REQ $sdreq WORKDIR / +# TODO: Optimize image size + COPY entrypoint.sh anaconda.sh . SHELL ["/bin/bash", "-c"] -# Update and apt +# Update and apt 446 MB RUN apt update && apt upgrade -y \ && apt install -y \ git \ @@ -18,15 +20,15 @@ RUN apt update && apt upgrade -y \ python3 \ wget -# install Anaconda or Miniconda +# install Anaconda or Miniconda 610 MB RUN bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash -# SD repo +# SD repo 105 MB RUN git clone $GITHUB_STABLE_DIFFUSION WORKDIR /stable-diffusion -# SD env +# SD env 2.3 GB !!! RUN PIP_EXISTS_ACTION="w" \ # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. && CONDA_SUBDIR="osx-arm64" \ @@ -35,18 +37,18 @@ RUN PIP_EXISTS_ACTION="w" \ && pip3 install -r $SD_REQ \ && mkdir models/ldm/stable-diffusion-v1 -# Face restoration prerequisites +# Face restoration prerequisites 200 MB RUN apt install -y libgl1-mesa-glx libglib2.0-0 - # by default expected in a sibling directory to stable-diffusion WORKDIR / -# Face restoreation repo +# Face restoreation repo 12 MB +# by default expected in a sibling directory to stable-diffusion RUN git clone https://github.com/TencentARC/GFPGAN.git WORKDIR /GFPGAN -# Face restoration env +# Face restoration env 608 MB RUN pip3 install basicsr facexlib \ && pip3 install -r requirements.txt \ && python3 setup.py develop \ @@ -57,7 +59,7 @@ RUN pip3 install basicsr facexlib \ WORKDIR /stable-diffusion -# Preload models +# Preload models 2 GB RUN python3 scripts/preload_models.py ENTRYPOINT ["/entrypoint.sh"] From 585b47fdd1d28b8e566d5fa9486d5a79d5b75aa4 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 15:36:09 -0500 Subject: [PATCH 023/238] Docker platform now configurable. No longer Mac-specific --- README-Mac-Docker.md | 11 ++++++++--- docker-build/Dockerfile | 11 +++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index b2f9a93a8d..3c8d907807 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -4,7 +4,7 @@ Table of Contents * [Step 1 - Get the Model](#step-1---get-the-model) * [Step 2 - Installation](#step-2---installation) - * [Option A - On a Linux container with Docker for Apple silicon](#option-a---on-a-linux-container-with-docker-for-apple-silicon) + * [Option A - On a Linux container with Docker](#option-a---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites) * [Setup](#setup) * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) @@ -23,7 +23,8 @@ You'll need to create an account but it's quick and free. # Step 2 - Installation -## Option A - On a Linux container with Docker for Apple silicon +## Option A - On a Linux container +This example uses a Mac M2 but you can specify the platform and architecture as parameters when building the image and running the container. You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. ### Prerequisites @@ -59,9 +60,13 @@ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O a ``` Build the Docker image. Give it any tag ```-t``` that you want. -Tip: Make sure your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. +Tip: Check that your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. +Base image will be arm64v8/debian on a macOS host. +```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. M1/M2 is ARM-based. On macOS you could also conda install ```nomkl``` but setting the environment appropriately is cleaner. ```Shell docker build -t santisbon/stable-diffusion \ +--platform linux/arm64 \ +--build-arg condaarch="osx-arm64" \ --build-arg gsd=$GITHUB_STABLE_DIFFUSION \ --build-arg sdreq="requirements-linux-arm64.txt" \ . diff --git a/docker-build/Dockerfile b/docker-build/Dockerfile index 3f57b14e18..05dcb822cc 100644 --- a/docker-build/Dockerfile +++ b/docker-build/Dockerfile @@ -1,10 +1,14 @@ -FROM arm64v8/debian +FROM debian ARG gsd ENV GITHUB_STABLE_DIFFUSION $gsd + ARG sdreq="requirements-linux-arm64.txt" ENV SD_REQ $sdreq +ARG condaarch +ENV ARCH $condaarch + WORKDIR / # TODO: Optimize image size @@ -30,10 +34,9 @@ WORKDIR /stable-diffusion # SD env 2.3 GB !!! RUN PIP_EXISTS_ACTION="w" \ - # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. - && CONDA_SUBDIR="osx-arm64" \ - # Create the environment, activate it, install requirements. + && CONDA_SUBDIR=$ARCH \ && source ~/.bashrc && conda create -y --name ldm && conda activate ldm \ + && conda config --env --set subdir $ARCH \ && pip3 install -r $SD_REQ \ && mkdir models/ldm/stable-diffusion-v1 From 443a4ad87c1531b958ba94de190b2e664a80636b Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 15:57:32 -0500 Subject: [PATCH 024/238] [Documentation] Updates on Docker platform --- README-Mac-Docker.md => README-Docker.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename README-Mac-Docker.md => README-Docker.md (92%) diff --git a/README-Mac-Docker.md b/README-Docker.md similarity index 92% rename from README-Mac-Docker.md rename to README-Docker.md index 3c8d907807..ff311b064a 100644 --- a/README-Mac-Docker.md +++ b/README-Docker.md @@ -4,7 +4,7 @@ Table of Contents * [Step 1 - Get the Model](#step-1---get-the-model) * [Step 2 - Installation](#step-2---installation) - * [Option A - On a Linux container with Docker](#option-a---on-a-linux-container-with-docker-for-apple-silicon) + * [Option A - On a Linux container](#option-a---on-a-linux-container) * [Prerequisites](#prerequisites) * [Setup](#setup) * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) @@ -24,8 +24,9 @@ You'll need to create an account but it's quick and free. # Step 2 - Installation ## Option A - On a Linux container -This example uses a Mac M2 but you can specify the platform and architecture as parameters when building the image and running the container. -You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. +This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. It provides a reliable way to generate a build and deploy it. It also uses a Docker volume to store the largest model file as a first step in decoupling storage and compute. Future enhancements will do this for other model files and assets. The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. + +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced compared with running it directly on macOS but for development purposes it's fine. ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#docker) @@ -62,7 +63,7 @@ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O a Build the Docker image. Give it any tag ```-t``` that you want. Tip: Check that your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. Base image will be arm64v8/debian on a macOS host. -```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. M1/M2 is ARM-based. On macOS you could also conda install ```nomkl``` but setting the environment appropriately is cleaner. +```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. On macOS you could also conda install ```nomkl``` but setting the environment appropriately is cleaner. ```Shell docker build -t santisbon/stable-diffusion \ --platform linux/arm64 \ From 3f8a289e9b24a54ecd419cfff07a964e7964b480 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 16:23:04 -0500 Subject: [PATCH 025/238] [Documentation] Clarification on which conda installer to use --- README-Docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README-Docker.md b/README-Docker.md index ff311b064a..87ea047aae 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -56,13 +56,13 @@ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion/docker-build chmod +x entrypoint.sh -# download the Miniconda installer. We'll need it at build time. +# Download the Miniconda installer. We'll need it at build time. +# Replace the URL with the version matching your system. wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh ``` Build the Docker image. Give it any tag ```-t``` that you want. Tip: Check that your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. -Base image will be arm64v8/debian on a macOS host. ```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. On macOS you could also conda install ```nomkl``` but setting the environment appropriately is cleaner. ```Shell docker build -t santisbon/stable-diffusion \ From b20f2bcd7e1c5debc234de15b5bca919ba7abdb2 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 19:09:05 -0500 Subject: [PATCH 026/238] [Documentation] Why containers? --- README-Docker.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README-Docker.md b/README-Docker.md index 87ea047aae..d210954c5e 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -5,6 +5,7 @@ Table of Contents * [Step 1 - Get the Model](#step-1---get-the-model) * [Step 2 - Installation](#step-2---installation) * [Option A - On a Linux container](#option-a---on-a-linux-container) + * [Why containers?](#why-containers) * [Prerequisites](#prerequisites) * [Setup](#setup) * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) @@ -17,6 +18,7 @@ Table of Contents * [Web Interface](#web-interface) * [Notes](#notes) + # Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. @@ -24,9 +26,13 @@ You'll need to create an account but it's quick and free. # Step 2 - Installation ## Option A - On a Linux container -This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. It provides a reliable way to generate a build and deploy it. It also uses a Docker volume to store the largest model file as a first step in decoupling storage and compute. Future enhancements will do this for other model files and assets. The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. -You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced compared with running it directly on macOS but for development purposes it's fine. +### Why containers? +They provide a flexible, reliable way to build and deploy Stable Diffusion. We also use a Docker volume to store the largest model file and image outputs as a first step in decoupling storage and compute. Future enhancements will do this for other model files and assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important. + +This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. + +The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to en environment with NVIDIA GPUs on-premises or in the cloud. ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#docker) From c705ff5e72b5047820586688140cbc4e5e1cd176 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 19:26:42 -0500 Subject: [PATCH 027/238] [Documentation] Fix typo --- README-Docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-Docker.md b/README-Docker.md index d210954c5e..59cec6375b 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -32,7 +32,7 @@ They provide a flexible, reliable way to build and deploy Stable Diffusion. We a This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. -The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to en environment with NVIDIA GPUs on-premises or in the cloud. +The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to an environment with NVIDIA GPUs on-premises or in the cloud. ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#docker) From 012c0dfdebb43c71d6c8be558e746f57c6993416 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sun, 11 Sep 2022 22:11:34 -0500 Subject: [PATCH 028/238] [Documentation] Cleanup --- README-Docker.md | 80 ++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/README-Docker.md b/README-Docker.md index 59cec6375b..ce06e27105 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -18,7 +18,6 @@ Table of Contents * [Web Interface](#web-interface) * [Notes](#notes) - # Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. @@ -43,18 +42,18 @@ Create a Docker volume for the downloaded model file docker volume create my-vol ``` -Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +Copy the model file (we'll need it at run time) to the Docker volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. ```Shell -docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine -# Copy the model file to the Docker volume. We'll need it at run time. cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` ### Setup +Set the fork you want to use. +Download the Miniconda installer (we'll need it at build time). Replace the URL with the version matching your system. ```Shell -# Set the fork you want to use. GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" cd ~ @@ -62,14 +61,13 @@ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion/docker-build chmod +x entrypoint.sh -# Download the Miniconda installer. We'll need it at build time. -# Replace the URL with the version matching your system. + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh ``` Build the Docker image. Give it any tag ```-t``` that you want. Tip: Check that your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. -```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. On macOS you could also conda install ```nomkl``` but setting the environment appropriately is cleaner. +```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. ```Shell docker build -t santisbon/stable-diffusion \ --platform linux/arm64 \ @@ -107,8 +105,8 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ### Setup +Set the fork you want to use. ```Shell -# Set the fork you want to use. GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" git clone $GITHUB_STABLE_DIFFUSION @@ -116,15 +114,19 @@ cd stable-diffusion mkdir -p models/ldm/stable-diffusion-v1/ ``` +When the pip3 path exists, it will ```w```ipe. + +The subdir env variable restricts conda to only use ARM packages while creating the env (M1/M2 is ARM-based). You could also ```conda install nomkl``` but setting the environment appropriately is cleaner. + +```conda config``` will write to the active environment's (```ldm```) configuration file and set ```subdir``` to the desired value permanently. ```Shell PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt -# When path exists, pip3 will (w)ipe. -# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. PIP_EXISTS_ACTION="w" CONDA_SUBDIR="osx-arm64" conda env create -f environment-mac.yaml && conda activate ldm +conda config --env --set subdir "osx-arm64" ``` You can verify you're in the virtual environment by looking at which executable you're getting: @@ -132,19 +134,20 @@ You can verify you're in the virtual environment by looking at which executable type python3 ``` -Face Restoration and Upscaling +**Face Restoration and Upscaling** +By default this repo is expected in a directory at the same level as stable-diffusion. +We'll need ```basicsr``` for training and inference and ```facexlib```, a face detection / face restoration helper. +Also ```realesrgan``` to enhance the background (non-face) regions and do upscaling. +Lastly, we'll get a pre-trained model needed for face restoration. ```Shell - -# by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN -# basicsr: used for training and inference. facexlib: face detection / face restoration helper. pip3 install basicsr facexlib \ && pip3 install -r requirements.txt python3 setup.py develop -pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling -# pre-trained model needed for face restoration +pip3 install realesrgan + wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion @@ -158,16 +161,14 @@ python3 scripts/preload_models.py # Step 3 - Usage (time to have fun) ## Startup -If you're on a Linux container the ```dream``` script is automatically started and the output dir set to the Docker volume you created earlier. - -If you're directly on macOS follow these startup instructions. -With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: -Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. +If you're on a **Linux container** the ```dream``` script is **automatically started** and the output dir set to the Docker volume you created earlier. +If you're **directly on macOS follow these startup instructions**. +With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts ```txt2img``` and ```img2img```: +Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. +By default the images are saved in ```outputs/img-samples/```. ```Shell -# If on Macbook -python3 scripts/dream.py --full_precision -# By default the images are saved in outputs/img-samples/. +python3 scripts/dream.py --full_precision ``` You'll get the script's prompt. You can see available options or quit. @@ -185,36 +186,41 @@ dream> The hulk fighting with sheldon cooper -s5 -n1 dream> "woman closeup highly detailed" -s 150 # Reuse previous seed and apply face restoration (if you installed GFPGAN) dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 -# TODO: example for upscaling. ``` + You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. -The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). +The ```-U``` option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). If you're on a container the output is set to the Docker volume. You can copy it wherever you want. You can download it from the Docker Desktop app, Volumes, my-vol, data. -Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: +Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name. + +On your host Mac (you can use the name of any container that mounted the volume): ```Shell -# On your host Macbook (you can use the name of any container that mounted the volume) docker cp dummy:/data/000001.928403745.png /Users//Pictures ``` ## Image to Image You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing. -Strength is a value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely. + +```strength``` is a value between 0.0 and 1.0 that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely. + Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256. -If you're on a docker container, copy your input image into the Docker volume +If you're on a Docker container, copy your input image into the Docker volume ```Shell docker cp /Users//Pictures/sketch-mountains-input.jpg dummy:/data/ ``` -Try it out generating an image (or 4). -The ```dream``` script needs absolute paths to find the image so don't use ```~```. +Try it out generating an image (or more). The ```dream``` script needs absolute paths to find the image so don't use ```~```. + +If you're on your Mac +```Shell +dream> "A fantasy landscape, trending on artstation" -I /Users//Pictures/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n4 +``` +If you're on a Linux container on your Mac ```Shell -# If you're on your Macbook -dream> "A fantasy landscape, trending on artstation" -I /Users//Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 -# If you're on a Linux container on your Macbook -dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n1 +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 50 -n1 ``` ## Web Interface From dbf2c63c9035ed692ccdc5c9871772686325a6ec Mon Sep 17 00:00:00 2001 From: Travco Date: Mon, 12 Sep 2022 15:37:26 -0400 Subject: [PATCH 029/238] Add Embiggen automation to upscale-cut-img2img-stitch and achieve high res without extra VRAM (#437) * Add Embiggen automation * Make embiggen_tiles masking more intelligent and count from one (at least for the user), rewrite sections of Embiggen README, fix various typos throughout README * drop duplicate log message --- README.md | 3 +- ldm/dream/generator/embiggen.py | 403 ++++++++++++++++++++++++++++++++ ldm/dream/generator/img2img.py | 2 +- ldm/dream/pngwriter.py | 4 + ldm/generate.py | 19 ++ scripts/dream.py | 16 +- 6 files changed, 443 insertions(+), 4 deletions(-) create mode 100644 ldm/dream/generator/embiggen.py diff --git a/README.md b/README.md index 1737e6515b..49d57811fe 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ report bugs and make feature requests. Be sure to use the provided templates. They will help aid diagnose issues faster._ # **Table of Contents** - 1. [Installation](#installation) 2. [Major Features](#features) 3. [Changelog](#latest-changes) @@ -134,7 +133,7 @@ To run in full-precision mode, start `dream.py` with the - Works on M1 Apple hardware. - Multiple bug fixes. -For older changelogs, please visit **[CHANGELOGS](docs/CHANGELOG.md)**. +For older changelogs, please visit **[CHANGELOGS](docs/CHANGELOG.md)**. # Troubleshooting diff --git a/ldm/dream/generator/embiggen.py b/ldm/dream/generator/embiggen.py new file mode 100644 index 0000000000..cb9c029a66 --- /dev/null +++ b/ldm/dream/generator/embiggen.py @@ -0,0 +1,403 @@ +''' +ldm.dream.generator.embiggen descends from ldm.dream.generator +and generates with ldm.dream.generator.img2img +''' + +import torch +import numpy as np +from PIL import Image +from ldm.dream.generator.base import Generator +from ldm.models.diffusion.ddim import DDIMSampler +from ldm.dream.generator.img2img import Img2Img + +class Embiggen(Generator): + def __init__(self,model): + super().__init__(model) + self.init_latent = None + + @torch.no_grad() + def get_make_image( + self, + prompt, + sampler, + steps, + cfg_scale, + ddim_eta, + conditioning, + init_img, + strength, + width, + height, + embiggen, + embiggen_tiles, + step_callback=None, + **kwargs + ): + """ + Returns a function returning an image derived from the prompt and multi-stage twice-baked potato layering over the img2img on the initial image + Return value depends on the seed at the time you call it + """ + # Construct embiggen arg array, and sanity check arguments + if embiggen == None: # embiggen can also be called with just embiggen_tiles + embiggen = [1.0] # If not specified, assume no scaling + elif embiggen[0] < 0 : + embiggen[0] = 1.0 + print('>> Embiggen scaling factor cannot be negative, fell back to the default of 1.0 !') + if len(embiggen) < 2: + embiggen.append(0.75) + elif embiggen[1] > 1.0 or embiggen[1] < 0 : + embiggen[1] = 0.75 + print('>> Embiggen upscaling strength for ESRGAN must be between 0 and 1, fell back to the default of 0.75 !') + if len(embiggen) < 3: + embiggen.append(0.25) + elif embiggen[2] < 0 : + embiggen[2] = 0.25 + print('>> Overlap size for Embiggen must be a positive ratio between 0 and 1 OR a number of pixels, fell back to the default of 0.25 !') + + # Convert tiles from their user-freindly count-from-one to count-from-zero, because we need to do modulo math + # and then sort them, because... people. + if embiggen_tiles: + embiggen_tiles = list(map(lambda n: n-1, embiggen_tiles)) + embiggen_tiles.sort() + + # Prep img2img generator, since we wrap over it + gen_img2img = Img2Img(self.model) + + # Open original init image (not a tensor) to manipulate + initsuperimage = Image.open(init_img) + + with Image.open(init_img) as img: + initsuperimage = img.convert('RGB') + + # Size of the target super init image in pixels + initsuperwidth, initsuperheight = initsuperimage.size + + # Increase by scaling factor if not already resized, using ESRGAN as able + if embiggen[0] != 1.0: + initsuperwidth = round(initsuperwidth*embiggen[0]) + initsuperheight = round(initsuperheight*embiggen[0]) + if embiggen[1] > 0: # No point in ESRGAN upscaling if strength is set zero + from ldm.gfpgan.gfpgan_tools import ( + real_esrgan_upscale, + ) + print(f'>> ESRGAN upscaling init image prior to cutting with Embiggen with strength {embiggen[1]}') + if embiggen[0] > 2: + initsuperimage = real_esrgan_upscale( + initsuperimage, + embiggen[1], # upscale strength + 4, # upscale scale + self.seed, + ) + else: + initsuperimage = real_esrgan_upscale( + initsuperimage, + embiggen[1], # upscale strength + 2, # upscale scale + self.seed, + ) + # We could keep recursively re-running ESRGAN for a requested embiggen[0] larger than 4x + # but from personal experiance it doesn't greatly improve anything after 4x + # Resize to target scaling factor resolution + initsuperimage = initsuperimage.resize((initsuperwidth, initsuperheight), Image.Resampling.LANCZOS) + + # Use width and height as tile widths and height + # Determine buffer size in pixels + if embiggen[2] < 1: + if embiggen[2] < 0: + embiggen[2] = 0 + overlap_size_x = round(embiggen[2] * width) + overlap_size_y = round(embiggen[2] * height) + else: + overlap_size_x = round(embiggen[2]) + overlap_size_y = round(embiggen[2]) + + # With overall image width and height known, determine how many tiles we need + def ceildiv(a, b): + return -1 * (-a // b) + + # X and Y needs to be determined independantly (we may have savings on one based on the buffer pixel count) + # (initsuperwidth - width) is the area remaining to the right that we need to layers tiles to fill + # (width - overlap_size_x) is how much new we can fill with a single tile + emb_tiles_x = 1 + emb_tiles_y = 1 + if (initsuperwidth - width) > 0: + emb_tiles_x = ceildiv(initsuperwidth - width, width - overlap_size_x) + 1 + if (initsuperheight - height) > 0: + emb_tiles_y = ceildiv(initsuperheight - height, height - overlap_size_y) + 1 + # Sanity + assert emb_tiles_x > 1 or emb_tiles_y > 1, f'ERROR: Based on the requested dimensions of {initsuperwidth}x{initsuperheight} and tiles of {width}x{height} you don\'t need to Embiggen! Check your arguments.' + + # Prep alpha layers -------------- + # https://stackoverflow.com/questions/69321734/how-to-create-different-transparency-like-gradient-with-python-pil + # agradientL is Left-side transparent + agradientL = Image.linear_gradient('L').rotate(90).resize((overlap_size_x, height)) + # agradientT is Top-side transparent + agradientT = Image.linear_gradient('L').resize((width, overlap_size_y)) + # radial corner is the left-top corner, made full circle then cut to just the left-top quadrant + agradientC = Image.new('L', (256, 256)) + for y in range(256): + for x in range(256): + #Find distance to lower right corner (numpy takes arrays) + distanceToLR = np.sqrt([(255 - x) ** 2 + (255 - y) ** 2])[0] + #Clamp values to max 255 + if distanceToLR > 255: + distanceToLR = 255 + #Place the pixel as invert of distance + agradientC.putpixel((x, y), int(255 - distanceToLR)) + + # Create alpha layers default fully white + alphaLayerL = Image.new("L", (width, height), 255) + alphaLayerT = Image.new("L", (width, height), 255) + alphaLayerLTC = Image.new("L", (width, height), 255) + # Paste gradients into alpha layers + alphaLayerL.paste(agradientL, (0, 0)) + alphaLayerT.paste(agradientT, (0, 0)) + alphaLayerLTC.paste(agradientL, (0, 0)) + alphaLayerLTC.paste(agradientT, (0, 0)) + alphaLayerLTC.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0)) + + if embiggen_tiles: + # Individual unconnected sides + alphaLayerR = Image.new("L", (width, height), 255) + alphaLayerR.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) + alphaLayerB = Image.new("L", (width, height), 255) + alphaLayerB.paste(agradientT.rotate(180), (0, height - overlap_size_y)) + alphaLayerTB = Image.new("L", (width, height), 255) + alphaLayerTB.paste(agradientT, (0, 0)) + alphaLayerTB.paste(agradientT.rotate(180), (0, height - overlap_size_y)) + alphaLayerLR = Image.new("L", (width, height), 255) + alphaLayerLR.paste(agradientL, (0, 0)) + alphaLayerLR.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) + + # Sides and corner Layers + alphaLayerRBC = Image.new("L", (width, height), 255) + alphaLayerRBC.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) + alphaLayerRBC.paste(agradientT.rotate(180), (0, height - overlap_size_y)) + alphaLayerRBC.paste(agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, height - overlap_size_y)) + alphaLayerLBC = Image.new("L", (width, height), 255) + alphaLayerLBC.paste(agradientL, (0, 0)) + alphaLayerLBC.paste(agradientT.rotate(180), (0, height - overlap_size_y)) + alphaLayerLBC.paste(agradientC.rotate(90).resize((overlap_size_x, overlap_size_y)), (0, height - overlap_size_y)) + alphaLayerRTC = Image.new("L", (width, height), 255) + alphaLayerRTC.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) + alphaLayerRTC.paste(agradientT, (0, 0)) + alphaLayerRTC.paste(agradientC.rotate(270).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, 0)) + + # All but X layers + alphaLayerABT = Image.new("L", (width, height), 255) + alphaLayerABT.paste(alphaLayerLBC, (0, 0)) + alphaLayerABT.paste(agradientL.rotate(180), (width - overlap_size_x, 0)) + alphaLayerABT.paste(agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, height - overlap_size_y)) + alphaLayerABL = Image.new("L", (width, height), 255) + alphaLayerABL.paste(alphaLayerRTC, (0, 0)) + alphaLayerABL.paste(agradientT.rotate(180), (0, height - overlap_size_y)) + alphaLayerABL.paste(agradientC.rotate(180).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, height - overlap_size_y)) + alphaLayerABR = Image.new("L", (width, height), 255) + alphaLayerABR.paste(alphaLayerLBC, (0, 0)) + alphaLayerABR.paste(agradientT, (0, 0)) + alphaLayerABR.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0)) + alphaLayerABB = Image.new("L", (width, height), 255) + alphaLayerABB.paste(alphaLayerRTC, (0, 0)) + alphaLayerABB.paste(agradientL, (0, 0)) + alphaLayerABB.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0)) + + # All-around layer + alphaLayerAA = Image.new("L", (width, height), 255) + alphaLayerAA.paste(alphaLayerABT, (0, 0)) + alphaLayerAA.paste(agradientT, (0, 0)) + alphaLayerAA.paste(agradientC.resize((overlap_size_x, overlap_size_y)), (0, 0)) + alphaLayerAA.paste(agradientC.rotate(270).resize((overlap_size_x, overlap_size_y)), (width - overlap_size_x, 0)) + + # Clean up temporary gradients + del agradientL + del agradientT + del agradientC + + def make_image(x_T): + # Make main tiles ------------------------------------------------- + if embiggen_tiles: + print(f'>> Making {len(embiggen_tiles)} Embiggen tiles...') + else: + print(f'>> Making {(emb_tiles_x * emb_tiles_y)} Embiggen tiles ({emb_tiles_x}x{emb_tiles_y})...') + + emb_tile_store = [] + for tile in range(emb_tiles_x * emb_tiles_y): + # Determine if this is a re-run and replace + if embiggen_tiles and not tile in embiggen_tiles: + continue + # Get row and column entries + emb_row_i = tile // emb_tiles_x + emb_column_i = tile % emb_tiles_x + # Determine bounds to cut up the init image + # Determine upper-left point + if emb_column_i + 1 == emb_tiles_x: + left = initsuperwidth - width + else: + left = round(emb_column_i * (width - overlap_size_x)) + if emb_row_i + 1 == emb_tiles_y: + top = initsuperheight - height + else: + top = round(emb_row_i * (height - overlap_size_y)) + right = left + width + bottom = top + height + + # Cropped image of above dimension (does not modify the original) + newinitimage = initsuperimage.crop((left, top, right, bottom)) + # DEBUG: + # newinitimagepath = init_img[0:-4] + f'_emb_Ti{tile}.png' + # newinitimage.save(newinitimagepath) + + if embiggen_tiles: + print(f'Making tile #{tile + 1} ({embiggen_tiles.index(tile) + 1} of {len(embiggen_tiles)} requested)') + else: + print(f'Starting {tile + 1} of {(emb_tiles_x * emb_tiles_y)} tiles') + + # create a torch tensor from an Image + newinitimage = np.array(newinitimage).astype(np.float32) / 255.0 + newinitimage = newinitimage[None].transpose(0, 3, 1, 2) + newinitimage = torch.from_numpy(newinitimage) + newinitimage = 2.0 * newinitimage - 1.0 + newinitimage = newinitimage.to(self.model.device) + + tile_results = gen_img2img.generate( + prompt, + iterations = 1, + seed = self.seed, + sampler = sampler, + steps = steps, + cfg_scale = cfg_scale, + conditioning = conditioning, + ddim_eta = ddim_eta, + image_callback = None, # called only after the final image is generated + step_callback = step_callback, # called after each intermediate image is generated + width = width, + height = height, + init_img = init_img, # img2img doesn't need this, but it might in the future + init_image = newinitimage, # notice that init_image is different from init_img + mask_image = None, + strength = strength, + ) + + emb_tile_store.append(tile_results[0][0]) + # DEBUG (but, also has other uses), worth saving if you want tiles without a transparency overlap to manually composite + # emb_tile_store[-1].save(init_img[0:-4] + f'_emb_To{tile}.png') + del newinitimage + + # Sanity check we have them all + if len(emb_tile_store) == (emb_tiles_x * emb_tiles_y) or (embiggen_tiles != [] and len(emb_tile_store) == len(embiggen_tiles)): + outputsuperimage = Image.new("RGBA", (initsuperwidth, initsuperheight)) + if embiggen_tiles: + outputsuperimage.alpha_composite(initsuperimage.convert('RGBA'), (0, 0)) + for tile in range(emb_tiles_x * emb_tiles_y): + if embiggen_tiles: + if tile in embiggen_tiles: + intileimage = emb_tile_store.pop(0) + else: + continue + else: + intileimage = emb_tile_store[tile] + intileimage = intileimage.convert('RGBA') + # Get row and column entries + emb_row_i = tile // emb_tiles_x + emb_column_i = tile % emb_tiles_x + if emb_row_i == 0 and emb_column_i == 0 and not embiggen_tiles: + left = 0 + top = 0 + else: + # Determine upper-left point + if emb_column_i + 1 == emb_tiles_x: + left = initsuperwidth - width + else: + left = round(emb_column_i * (width - overlap_size_x)) + if emb_row_i + 1 == emb_tiles_y: + top = initsuperheight - height + else: + top = round(emb_row_i * (height - overlap_size_y)) + # Handle gradients for various conditions + # Handle emb_rerun case + if embiggen_tiles: + # top of image + if emb_row_i == 0: + if emb_column_i == 0: + if (tile+1) in embiggen_tiles: # Look-ahead right + if (tile+emb_tiles_x) not in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerB) + # Otherwise do nothing on this tile + elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only + intileimage.putalpha(alphaLayerR) + else: + intileimage.putalpha(alphaLayerRBC) + elif emb_column_i == emb_tiles_x - 1: + if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerL) + else: + intileimage.putalpha(alphaLayerLBC) + else: + if (tile+1) in embiggen_tiles: # Look-ahead right + if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerL) + else: + intileimage.putalpha(alphaLayerLBC) + elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only + intileimage.putalpha(alphaLayerLR) + else: + intileimage.putalpha(alphaLayerABT) + # bottom of image + elif emb_row_i == emb_tiles_y - 1: + if emb_column_i == 0: + if (tile+1) in embiggen_tiles: # Look-ahead right + intileimage.putalpha(alphaLayerT) + else: + intileimage.putalpha(alphaLayerRTC) + elif emb_column_i == emb_tiles_x - 1: + # No tiles to look ahead to + intileimage.putalpha(alphaLayerLTC) + else: + if (tile+1) in embiggen_tiles: # Look-ahead right + intileimage.putalpha(alphaLayerLTC) + else: + intileimage.putalpha(alphaLayerABB) + # vertical middle of image + else: + if emb_column_i == 0: + if (tile+1) in embiggen_tiles: # Look-ahead right + if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerT) + else: + intileimage.putalpha(alphaLayerTB) + elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only + intileimage.putalpha(alphaLayerRTC) + else: + intileimage.putalpha(alphaLayerABL) + elif emb_column_i == emb_tiles_x - 1: + if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerLTC) + else: + intileimage.putalpha(alphaLayerABR) + else: + if (tile+1) in embiggen_tiles: # Look-ahead right + if (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down + intileimage.putalpha(alphaLayerLTC) + else: + intileimage.putalpha(alphaLayerABR) + elif (tile+emb_tiles_x) in embiggen_tiles: # Look-ahead down only + intileimage.putalpha(alphaLayerABB) + else: + intileimage.putalpha(alphaLayerAA) + # Handle normal tiling case (much simpler - since we tile left to right, top to bottom) + else: + if emb_row_i == 0 and emb_column_i >= 1: + intileimage.putalpha(alphaLayerL) + elif emb_row_i >= 1 and emb_column_i == 0: + intileimage.putalpha(alphaLayerT) + else: + intileimage.putalpha(alphaLayerLTC) + # Layer tile onto final image + outputsuperimage.alpha_composite(intileimage, (left, top)) + else: + print(f'Error: could not find all Embiggen output tiles in memory? Something must have gone wrong with img2img generation.') + + # after internal loops and patching up return Embiggen image + return outputsuperimage + # end of function declaration + return make_image \ No newline at end of file diff --git a/ldm/dream/generator/img2img.py b/ldm/dream/generator/img2img.py index 242912d0eb..6a1561db6f 100644 --- a/ldm/dream/generator/img2img.py +++ b/ldm/dream/generator/img2img.py @@ -1,5 +1,5 @@ ''' -ldm.dream.generator.txt2img descends from ldm.dream.generator +ldm.dream.generator.img2img descends from ldm.dream.generator ''' import torch diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index 3f3a15891b..4e8bca2366 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -73,6 +73,10 @@ class PromptFormatter: switches.append(f'-G{opt.gfpgan_strength}') if opt.upscale: switches.append(f'-U {" ".join([str(u) for u in opt.upscale])}') + if opt.embiggen: + switches.append(f'-embiggen {" ".join([str(u) for u in opt.embiggen])}') + if opt.embiggen_tiles: + switches.append(f'-embiggen_tiles {" ".join([str(u) for u in opt.embiggen_tiles])}') if opt.variation_amount > 0: switches.append(f'-v{opt.variation_amount}') if opt.with_variations: diff --git a/ldm/generate.py b/ldm/generate.py index 8f67403633..89bbb470f5 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -205,6 +205,9 @@ class Generate: init_mask = None, fit = False, strength = None, + # these are specific to embiggen (which also relies on img2img args) + embiggen = None, + embiggen_tiles = None, # these are specific to GFPGAN/ESRGAN gfpgan_strength= 0, save_original = False, @@ -230,6 +233,8 @@ class Generate: image_callback // a function or method that will be called each time an image is generated with_variations // a weighted list [(seed_1, weight_1), (seed_2, weight_2), ...] of variations which should be applied before doing any generation variation_amount // optional 0-1 value to slerp from -S noise to random noise (allows variations on an image) + embiggen // scale factor relative to the size of the --init_img (-I), followed by ESRGAN upscaling strength (0-1.0), followed by minimum amount of overlap between tiles as a decimal ratio (0 - 1.0) or number of pixels + embiggen_tiles // list of tiles by number in order to process and replace onto the image e.g. `0 2 4` To use the step callback, define a function that receives two arguments: - Image GPU data @@ -274,6 +279,9 @@ class Generate: assert ( 0.0 <= variation_amount <= 1.0 ), '-v --variation_amount must be in [0.0, 1.0]' + assert ( + (embiggen == None and embiggen_tiles == None) or ((embiggen != None or embiggen_tiles != None) and init_img != None) + ), 'Embiggen requires an init/input image to be specified' # check this logic - doesn't look right if len(with_variations) > 0 or variation_amount > 1.0: @@ -310,6 +318,8 @@ class Generate: if (init_image is not None) and (mask_image is not None): generator = self._make_inpaint() + elif (embiggen != None or embiggen_tiles != None): + generator = self._make_embiggen() elif init_image is not None: generator = self._make_img2img() else: @@ -329,9 +339,12 @@ class Generate: step_callback = step_callback, # called after each intermediate image is generated width = width, height = height, + init_img = init_img, # embiggen needs to manipulate from the unmodified init_img init_image = init_image, # notice that init_image is different from init_img mask_image = mask_image, strength = strength, + embiggen = embiggen, + embiggen_tiles = embiggen_tiles, ) if upscale is not None or gfpgan_strength > 0: @@ -404,6 +417,12 @@ class Generate: from ldm.dream.generator.img2img import Img2Img self.generators['img2img'] = Img2Img(self.model) return self.generators['img2img'] + + def _make_embiggen(self): + if not self.generators.get('embiggen'): + from ldm.dream.generator.embiggen import Embiggen + self.generators['embiggen'] = Embiggen(self.model) + return self.generators['embiggen'] def _make_txt2img(self): if not self.generators.get('txt2img'): diff --git a/scripts/dream.py b/scripts/dream.py index 891a448bf2..8559c1b083 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -631,7 +631,7 @@ def create_cmd_parser(): nargs='+', default=None, type=float, - help='Scale factor (2, 4) for upscaling followed by upscaling strength (0-1.0). If strength not specified, defaults to 0.75' + help='Scale factor (2, 4) for upscaling final output followed by upscaling strength (0-1.0). If strength not specified, defaults to 0.75' ) parser.add_argument( '-save_orig', @@ -639,6 +639,20 @@ def create_cmd_parser(): action='store_true', help='Save original. Use it when upscaling to save both versions.', ) + parser.add_argument( + '-embiggen', + nargs='+', + default=None, + type=float, + help='Embiggen tiled img2img for higher resolution and detail without extra VRAM usage. Takes scale factor relative to the size of the --init_img (-I), followed by ESRGAN upscaling strength (0-1.0), followed by minimum amount of overlap between tiles as a decimal ratio (0 - 1.0) or number of pixels. ESRGAN strength defaults to 0.75, and overlap defaults to 0.25 . ESRGAN is used to upscale the init prior to cutting it into tiles/pieces to run through img2img and then stitch back togeather.', + ) + parser.add_argument( + '-embiggen_tiles', + nargs='+', + default=None, + type=int, + help='If while doing Embiggen we are altering only parts of the image, takes a list of tiles by number to process and replace onto the image e.g. `1 3 5`, useful for redoing problematic spots from a prior Embiggen run', + ) # variants is going to be superseded by a generalized "prompt-morph" function # parser.add_argument('-v','--variants',type=int,help="in img2img mode, the first generated image will get passed back to img2img to generate the requested number of variants") parser.add_argument( From 6665f4494f0360dbc653f44bddeef0431d6bbbf8 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 12 Sep 2022 15:46:07 -0400 Subject: [PATCH 030/238] Add documentation for Embiggen This was originally part of PR #437, but was inside README.md which was refactored. Now it is a standalone doc. --- README.md | 2 + docs/features/EMBIGGEN.md | 134 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 docs/features/EMBIGGEN.md diff --git a/README.md b/README.md index 49d57811fe..539715fae8 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ To run in full-precision mode, start `dream.py` with the - ## [GFPGAN and Real-ESRGAN Support](docs/features/UPSCALE.md) +- ## [Embiggen upscaling](docs/features/EMBIGGEN.md) + - ## [Seamless Tiling](docs/features/OTHER.md#seamless-tiling) - ## [Google Colab](docs/features/OTHER.md#google-colab) diff --git a/docs/features/EMBIGGEN.md b/docs/features/EMBIGGEN.md new file mode 100644 index 0000000000..70f35fe758 --- /dev/null +++ b/docs/features/EMBIGGEN.md @@ -0,0 +1,134 @@ +# **Embiggen -- upscale your images on limited memory machines** + +GFPGAN and Real-ESRGAN are both memory intensive. In order to avoid +crashes and memory overloads during the Stable Diffusion process, +these effects are applied after Stable Diffusion has completed its +work. + +In single image generations, you will see the output right away but +when you are using multiple iterations, the images will first be +generated and then upscaled and face restored after that process is +complete. While the image generation is taking place, you will still +be able to preview the base images. + +If you wish to stop during the image generation but want to upscale or +face restore a particular generated image, pass it again with the same +prompt and generated seed along with the `-U` and `-G` prompt +arguments to perform those actions. + +## Embiggen + +If you wanted to be able to do more (pixels) without running out of VRAM, +or you want to upscale with details that couldn't possibly appear +without the context of a prompt, this is the feature to try out. + +Embiggen automates the process of taking an init image, upscaling it, +cutting it into smaller tiles that slightly overlap, running all the +tiles through img2img to refine details with respect to the prompt, +and "stitching" the tiles back together into a cohesive image. + +It automatically computes how many tiles are needed, and so it can be fed +*ANY* size init image and perform Img2Img on it (though it will be run only +one tile at a time, which can cause problems, see the Note at the end). + +If you're familiar with "GoBig" (ala [progrock-stable](https://github.com/lowfuel/progrock-stable)) +it's similar to that, except it can work up to an arbitrarily large size +(instead of just 2x), with tile overlaps configurable as a ratio, and +has extra logic to re-run any number of the tile sub-sections of the image +if for example a small part of a huge run got messed up. + +**Usage** + +`-embiggen ` + +Takes a scaling factor relative to the size of the `--init_img` (`-I`), followed by +ESRGAN upscaling strength (0 - 1.0), followed by minimum amount of overlap +between tiles as a decimal ratio (0 - 1.0) *OR* a number of pixels. + +The scaling factor is how much larger than the `--init_img` the output +should be, and will multiply both x and y axis, so an image that is a +scaling factor of 3.0 has 3*3= 9 times as many pixels, and will take +(at least) 9 times as long (see overlap for why it might be +longer). If the `--init_img` is already the right size `-embiggen 1`, +and it can also be less than one if the init_img is too big. + +Esrgan_strength defaults to 0.75, and the overlap_ratio defaults to +0.25, both are optional. + + +Unlike Img2Img, the `--width` (`-W`) and `--height` (`-H`) arguments +do not control the size of the image as a whole, but the size of the +tiles used to Embiggen the image. + +ESRGAN is used to upscale the `--init_img` prior to cutting it into +tiles/pieces to run through img2img and then stitch back +together. Embiggen can be run without ESRGAN; just set the strength to +zero (e.g. `-embiggen 1.75 0`). The output of Embiggen can also be +upscaled after it's finished (`-U`). + +The overlap is the minimum that tiles will overlap with adjacent +tiles, specified as either a ratio or a number of pixels. How much the +tiles overlap determines the likelihood the tiling will be noticable, +really small overlaps (e.g. a couple of pixels) may produce noticeable +grid-like fuzzy distortions in the final stitched image. Though, as +the overlapping space doesn't contribute to making the image bigger, +and the larger the overlap the more tiles (and the more time) it will +take to finish. + +Because the overlapping parts of tiles don't "contribute" to +increasing size, every tile after the first in a row or column +effectively only covers an extra `1 - overlap_ratio` on each axis. If +the input/`--init_img` is same size as a tile, the ideal (for time) +scaling factors with the default overlap (0.25) are 1.75, 2.5, 3.25, +4.0 etc.. + +`-embiggen_tiles ` + +An advanced usage useful if you only want to alter parts of the image +while running Embiggen. It takes a list of tiles by number to run and +replace onto the initial image e.g. `1 3 5`. It's useful for either +fixing problem spots from a previous Embiggen run, or selectively +altering the prompt for sections of an image - for creative or +coherency reasons. + +Tiles are numbered starting with one, and left-to-right, +top-to-bottom. So, if you are generating a 3x3 tiled image, the +middle row would be `4 5 6`. + +**Example Usage** + +Running Embiggen with 512x512 tiles on an existing image, scaling up by a factor of 2.5x; +and doing the same again (default ESRGAN strength is 0.75, default overlap between tiles is 0.25): + +``` +dream > a photo of a forest at sunset -s 100 -W 512 -H 512 -I outputs/forest.png -f 0.4 -embiggen 2.5 +dream > a photo of a forest at sunset -s 100 -W 512 -H 512 -I outputs/forest.png -f 0.4 -embiggen 2.5 0.75 0.25 +``` + +If your starting image was also 512x512 this should have taken 9 tiles. + +If there weren't enough clouds in the sky of that forest you just made +(and that image is about 1280 pixels (512*2.5) wide A.K.A. three +512x512 tiles with 0.25 overlaps wide) we can replace that top row of +tiles: + +``` +dream> a photo of puffy clouds over a forest at sunset -s 100 -W 512 -H 512 -I outputs/000002.seed.png -f 0.5 -embiggen_tiles 1 2 3 +``` + +**Note** + +Because the same prompt is used on all the tiled images, and the model +doesn't have the context of anything outside the tile being run - it +can end up creating repeated pattern (also called 'motifs') across all +the tiles based on that prompt. The best way to combat this is +lowering the `--strength` (`-f`) to stay more true to the init image, +and increasing the number of steps so there is more compute-time to +create the detail. Anecdotally `--strength` 0.35-0.45 works pretty +well on most things. It may also work great in some examples even with +the `--strength` set high for patterns, landscapes, or subjects that +are more abstract. Because this is (relatively) fast, you can also +always create a few Embiggen'ed images and manually composite them to +preserve the best parts from each. + +Author: [Travco](https://github.com/travco) \ No newline at end of file From 0bc6779361ffb8a31e68a4a817a14c7769631290 Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Mon, 12 Sep 2022 23:55:21 +0300 Subject: [PATCH 031/238] Disable autocast for cpu to fix error. Remove unused precision arg. (#518) When running on just cpu (intel), a call to torch.layer_norm would error with RuntimeError: expected scalar type BFloat16 but found Float Fix buggy device handling in model.py. Tested with scripts/dream.py --full_precision on just cpu on intel laptop. Works but slow at ~10s/it. --- ldm/dream/devices.py | 5 +++-- ldm/generate.py | 2 -- ldm/modules/diffusionmodules/model.py | 6 ++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ldm/dream/devices.py b/ldm/dream/devices.py index 90bc9e97dd..3b85a7420c 100644 --- a/ldm/dream/devices.py +++ b/ldm/dream/devices.py @@ -13,8 +13,9 @@ def choose_torch_device() -> str: def choose_autocast_device(device): '''Returns an autocast compatible device from a torch device''' device_type = device.type # this returns 'mps' on M1 - # autocast only supports cuda or cpu - if device_type in ('cuda','cpu'): + if device_type == 'cuda': return device_type,autocast + elif device_type == 'cpu': + return device_type,nullcontext else: return 'cpu',nullcontext diff --git a/ldm/generate.py b/ldm/generate.py index 89bbb470f5..52c8846d80 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -111,7 +111,6 @@ class Generate: height = 512, sampler_name = 'k_lms', ddim_eta = 0.0, # deterministic - precision = 'autocast', full_precision = False, strength = 0.75, # default in scripts/img2img.py seamless = False, @@ -129,7 +128,6 @@ class Generate: self.sampler_name = sampler_name self.grid = grid self.ddim_eta = ddim_eta - self.precision = precision self.full_precision = True if choose_torch_device() == 'mps' else full_precision self.strength = strength self.seamless = seamless diff --git a/ldm/modules/diffusionmodules/model.py b/ldm/modules/diffusionmodules/model.py index 970f6aad8f..5880452d47 100644 --- a/ldm/modules/diffusionmodules/model.py +++ b/ldm/modules/diffusionmodules/model.py @@ -209,8 +209,7 @@ class AttnBlock(nn.Module): h_ = torch.zeros_like(k, device=q.device) - device_type = 'mps' if q.device.type == 'mps' else 'cuda' - if device_type == 'cuda': + if q.device.type == 'cuda': stats = torch.cuda.memory_stats(q.device) mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] @@ -612,9 +611,8 @@ class Decoder(nn.Module): del h3 # prepare for up sampling - device_type = 'mps' if h.device.type == 'mps' else 'cuda' gc.collect() - if device_type == 'cuda': + if h.device.type == 'cuda': torch.cuda.empty_cache() # upsampling From b9592ff2dcf425b9d881df84990274fc5b973b48 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 12 Sep 2022 16:55:39 -0400 Subject: [PATCH 032/238] document ability to use -ve numbers to retrieve previous seeds and image paths (#524) --- docs/features/CLI.md | 75 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/docs/features/CLI.md b/docs/features/CLI.md index d809d48841..e50c5833c9 100644 --- a/docs/features/CLI.md +++ b/docs/features/CLI.md @@ -1,17 +1,30 @@ # **Interactive Command-Line Interface** -The `dream.py` script, located in `scripts/dream.py`, provides an interactive interface to image generation similar to the "dream mothership" bot that Stable AI provided on its Discord server. +The `dream.py` script, located in `scripts/dream.py`, provides an +interactive interface to image generation similar to the "dream +mothership" bot that Stable AI provided on its Discord server. -Unlike the txt2img.py and img2img.py scripts provided in the original CompViz/stable-diffusion source code repository, the time-consuming initialization of the AI model initialization only happens once. After that image generation -from the command-line interface is very fast. +Unlike the txt2img.py and img2img.py scripts provided in the original +CompViz/stable-diffusion source code repository, the time-consuming +initialization of the AI model initialization only happens once. After +that image generation from the command-line interface is very fast. -The script uses the readline library to allow for in-line editing, command history (up and down arrows), autocompletion, and more. To help keep track of which prompts generated which images, the script writes a log file of image names and prompts to the selected output directory. +The script uses the readline library to allow for in-line editing, +command history (up and down arrows), autocompletion, and more. To +help keep track of which prompts generated which images, the script +writes a log file of image names and prompts to the selected output +directory. -In addition, as of version 1.02, it also writes the prompt into the PNG file's metadata where it can be retrieved using scripts/images2prompt.py +In addition, as of version 1.02, it also writes the prompt into the +PNG file's metadata where it can be retrieved using +scripts/images2prompt.py The script is confirmed to work on Linux, Windows and Mac systems. -_Note:_ This script runs from the command-line or can be used as a Web application. The Web GUI is currently rudimentary, but a much better replacement is on its way. +_Note:_ This script runs from the command-line or can be used as a Web +application. The Web GUI is currently rudimentary, but a much better +replacement is on its way. + ``` (ldm) ~/stable-diffusion$ python3 ./scripts/dream.py @@ -183,6 +196,56 @@ well as the --mask (-M) argument: | --init_mask | -M | None |Path to an image the same size as the initial_image, with areas for inpainting made transparent.| +# Shortcuts + +Since one so frequently refers back to a previously-generated seed or +image, dream.py provides an easy shortcut that avoids having to cut +and paste these values. + +Here's how it works. Say you generated 6 images of a man-eating snail: + +~~~~ +dream> man-eating snail -n6 +... +>> Usage stats: +>> 6 image(s) generated in 79.85s +>> Max VRAM used for this generation: 3.36G. Current VRAM utilization:2.21G +>> Max VRAM used since script start: 3.36G +Outputs: +[1] outputs/img-samples/000210.1414805682.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1414805682 +[2] outputs/img-samples/000210.3312885013.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S3312885013 +[3] outputs/img-samples/000210.1398528919.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1398528919 +[4] outputs/img-samples/000210.92626031.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S92626031 +[5] outputs/img-samples/000210.1733666373.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S1733666373 +[6] outputs/img-samples/000210.2453524229.png: "man-eating snail" -s50 -W512 -H512 -C7.5 -Ak_lms -S2453524229 +~~~~ + +The last image generated (with seed 2453524229) looks really good. So let's +pick that one for variation generation. Instead of cutting and pasting +the argument -S2453524229, we can simply refer to the most recent seed as +-1, and write: + +~~~~ +dream> man-eating snail -v0.1 -n10 -S-1 +>> Reusing previous seed 2453524229 +...etc... +~~~~ + +You can use -2 to refer to the second to last seed, -3 to the third to +last, etc. It works with both individual images and grids. However, +the numbering system only extends across the last group of images +generated and doesn't reach back to earlier commands. + +The initial image (-I or --init_img) argument works in a similar +way. To use the second-to-most-recent snail image as the initial +image for an img2img render, you could refer to it as -I-2: + +~~~~ +dream> glowing science-fiction snail -I -2 -n4 +>> Reusing previous image outputs/img-samples/000213.2150458613.png +...etc... +~~~~ + # Command-line editing and completion If you are on a Macintosh or Linux machine, the command-line offers From dedf8a36924787731d29418521a0619c3d6d147c Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Tue, 13 Sep 2022 00:39:06 +0300 Subject: [PATCH 033/238] Remove pointless del statements in diffusionmodules.model. (#520) --- ldm/modules/diffusionmodules/model.py | 70 ++++++++------------------- 1 file changed, 19 insertions(+), 51 deletions(-) diff --git a/ldm/modules/diffusionmodules/model.py b/ldm/modules/diffusionmodules/model.py index 5880452d47..a3598c40ef 100644 --- a/ldm/modules/diffusionmodules/model.py +++ b/ldm/modules/diffusionmodules/model.py @@ -121,30 +121,17 @@ class ResnetBlock(nn.Module): padding=0) def forward(self, x, temb): - h1 = x - h2 = self.norm1(h1) - del h1 - - h3 = nonlinearity(h2) - del h2 - - h4 = self.conv1(h3) - del h3 + h = self.norm1(x) + h = nonlinearity(h) + h = self.conv1(h) if temb is not None: - h4 = h4 + self.temb_proj(nonlinearity(temb))[:,:,None,None] + h = h + self.temb_proj(nonlinearity(temb))[:,:,None,None] - h5 = self.norm2(h4) - del h4 - - h6 = nonlinearity(h5) - del h5 - - h7 = self.dropout(h6) - del h6 - - h8 = self.conv2(h7) - del h7 + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) if self.in_channels != self.out_channels: if self.use_conv_shortcut: @@ -152,7 +139,7 @@ class ResnetBlock(nn.Module): else: x = self.nin_shortcut(x) - return x + h8 + return x + h class LinAttnBlock(LinearAttention): """to match AttnBlock usage""" @@ -598,17 +585,12 @@ class Decoder(nn.Module): temb = None # z to block_in - h1 = self.conv_in(z) + h = self.conv_in(z) # middle - h2 = self.mid.block_1(h1, temb) - del h1 - - h3 = self.mid.attn_1(h2) - del h2 - - h = self.mid.block_2(h3, temb) - del h3 + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) # prepare for up sampling gc.collect() @@ -620,33 +602,19 @@ class Decoder(nn.Module): for i_block in range(self.num_res_blocks+1): h = self.up[i_level].block[i_block](h, temb) if len(self.up[i_level].attn) > 0: - t = h - h = self.up[i_level].attn[i_block](t) - del t - + h = self.up[i_level].attn[i_block](h) if i_level != 0: - t = h - h = self.up[i_level].upsample(t) - del t + h = self.up[i_level].upsample(h) # end if self.give_pre_end: return h - h1 = self.norm_out(h) - del h - - h2 = nonlinearity(h1) - del h1 - - h = self.conv_out(h2) - del h2 - + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) if self.tanh_out: - t = h - h = torch.tanh(t) - del t - + h = torch.tanh(h) return h From a18d0b9ef1feddcf061b861f356c7711ca40e9bb Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 12 Sep 2022 18:12:44 -0400 Subject: [PATCH 034/238] Squashed commit of the following: commit 9b28c65e4b0526956fd90ad86e2118536e4eaa9b Author: Lincoln Stein Date: Mon Sep 12 18:10:27 2022 -0400 revert inadvertent change of conda env name (#528) --- environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index db754a9185..30920eee9a 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,4 +1,4 @@ -name: sd-ldm +name: ldm channels: - pytorch - defaults From 654ec17000d397dc6d1c4e5371934bfae1c30a3a Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Mon, 12 Sep 2022 21:43:15 -0500 Subject: [PATCH 035/238] Remove Apple silicon section --- README-Docker.md | 87 ------------------------------------------------ 1 file changed, 87 deletions(-) diff --git a/README-Docker.md b/README-Docker.md index ce06e27105..8fddf986c5 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -1,23 +1,3 @@ - -Table of Contents -================= - -* [Step 1 - Get the Model](#step-1---get-the-model) -* [Step 2 - Installation](#step-2---installation) - * [Option A - On a Linux container](#option-a---on-a-linux-container) - * [Why containers?](#why-containers) - * [Prerequisites](#prerequisites) - * [Setup](#setup) - * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) - * [Prerequisites](#prerequisites-1) - * [Setup](#setup-1) -* [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) - * [Startup](#startup) - * [Text to Image](#text-to-image) - * [Image to Image](#image-to-image) - * [Web Interface](#web-interface) - * [Notes](#notes) - # Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. @@ -91,73 +71,6 @@ santisbon/stable-diffusion ``` Tip: Make sure you've created the Docker volume (above) -## Option B - Directly on Apple silicon -For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). - -### Prerequisites -Install the latest versions of macOS, [Homebrew](https://brew.sh/), [Python](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#python), and [Git](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#git). - -```Shell -brew install cmake protobuf rust -brew install --cask miniconda -conda init zsh && source ~/.zshrc # or bash and .bashrc -``` - -### Setup - -Set the fork you want to use. -```Shell -GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" - -git clone $GITHUB_STABLE_DIFFUSION -cd stable-diffusion -mkdir -p models/ldm/stable-diffusion-v1/ -``` - -When the pip3 path exists, it will ```w```ipe. - -The subdir env variable restricts conda to only use ARM packages while creating the env (M1/M2 is ARM-based). You could also ```conda install nomkl``` but setting the environment appropriately is cleaner. - -```conda config``` will write to the active environment's (```ldm```) configuration file and set ```subdir``` to the desired value permanently. -```Shell -PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt -ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt - -PIP_EXISTS_ACTION="w" -CONDA_SUBDIR="osx-arm64" -conda env create -f environment-mac.yaml && conda activate ldm -conda config --env --set subdir "osx-arm64" -``` - -You can verify you're in the virtual environment by looking at which executable you're getting: -```Shell -type python3 -``` - -**Face Restoration and Upscaling** -By default this repo is expected in a directory at the same level as stable-diffusion. -We'll need ```basicsr``` for training and inference and ```facexlib```, a face detection / face restoration helper. -Also ```realesrgan``` to enhance the background (non-face) regions and do upscaling. -Lastly, we'll get a pre-trained model needed for face restoration. -```Shell -cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN - -pip3 install basicsr facexlib \ -&& pip3 install -r requirements.txt - -python3 setup.py develop -pip3 install realesrgan - -wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models - -cd ../stable-diffusion -``` - -Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. Used by the core functionality and by GFPGAN/Real-ESRGAN. -```Shell -python3 scripts/preload_models.py -``` - # Step 3 - Usage (time to have fun) ## Startup From 82a223c5f6299eda63e4a087da919175187456e1 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Mon, 12 Sep 2022 21:52:23 -0500 Subject: [PATCH 036/238] Remove Apple silicon section --- README-Docker.md | 69 ------------------------------------------------ 1 file changed, 69 deletions(-) diff --git a/README-Docker.md b/README-Docker.md index ce06e27105..24cdd2ec58 100644 --- a/README-Docker.md +++ b/README-Docker.md @@ -8,9 +8,6 @@ Table of Contents * [Why containers?](#why-containers) * [Prerequisites](#prerequisites) * [Setup](#setup) - * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) - * [Prerequisites](#prerequisites-1) - * [Setup](#setup-1) * [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) * [Startup](#startup) * [Text to Image](#text-to-image) @@ -91,72 +88,6 @@ santisbon/stable-diffusion ``` Tip: Make sure you've created the Docker volume (above) -## Option B - Directly on Apple silicon -For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). - -### Prerequisites -Install the latest versions of macOS, [Homebrew](https://brew.sh/), [Python](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#python), and [Git](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#git). - -```Shell -brew install cmake protobuf rust -brew install --cask miniconda -conda init zsh && source ~/.zshrc # or bash and .bashrc -``` - -### Setup - -Set the fork you want to use. -```Shell -GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" - -git clone $GITHUB_STABLE_DIFFUSION -cd stable-diffusion -mkdir -p models/ldm/stable-diffusion-v1/ -``` - -When the pip3 path exists, it will ```w```ipe. - -The subdir env variable restricts conda to only use ARM packages while creating the env (M1/M2 is ARM-based). You could also ```conda install nomkl``` but setting the environment appropriately is cleaner. - -```conda config``` will write to the active environment's (```ldm```) configuration file and set ```subdir``` to the desired value permanently. -```Shell -PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt -ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt - -PIP_EXISTS_ACTION="w" -CONDA_SUBDIR="osx-arm64" -conda env create -f environment-mac.yaml && conda activate ldm -conda config --env --set subdir "osx-arm64" -``` - -You can verify you're in the virtual environment by looking at which executable you're getting: -```Shell -type python3 -``` - -**Face Restoration and Upscaling** -By default this repo is expected in a directory at the same level as stable-diffusion. -We'll need ```basicsr``` for training and inference and ```facexlib```, a face detection / face restoration helper. -Also ```realesrgan``` to enhance the background (non-face) regions and do upscaling. -Lastly, we'll get a pre-trained model needed for face restoration. -```Shell -cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN - -pip3 install basicsr facexlib \ -&& pip3 install -r requirements.txt - -python3 setup.py develop -pip3 install realesrgan - -wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models - -cd ../stable-diffusion -``` - -Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. Used by the core functionality and by GFPGAN/Real-ESRGAN. -```Shell -python3 scripts/preload_models.py -``` # Step 3 - Usage (time to have fun) From 00b002f731e4cd525130a06c07172baa7a576e07 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Mon, 12 Sep 2022 22:01:40 -0500 Subject: [PATCH 037/238] Move README-Docker to docs folder --- README-Docker.md => docs/installation/INSTALL_DOCKER.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README-Docker.md => docs/installation/INSTALL_DOCKER.md (100%) diff --git a/README-Docker.md b/docs/installation/INSTALL_DOCKER.md similarity index 100% rename from README-Docker.md rename to docs/installation/INSTALL_DOCKER.md From d4941ca833a1fd8d01f4932799b5407fae964bbd Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Mon, 12 Sep 2022 22:17:37 -0500 Subject: [PATCH 038/238] Doc cleanup --- docs/installation/INSTALL_DOCKER.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation/INSTALL_DOCKER.md b/docs/installation/INSTALL_DOCKER.md index 24cdd2ec58..86ba6dfb65 100644 --- a/docs/installation/INSTALL_DOCKER.md +++ b/docs/installation/INSTALL_DOCKER.md @@ -4,7 +4,7 @@ Table of Contents * [Step 1 - Get the Model](#step-1---get-the-model) * [Step 2 - Installation](#step-2---installation) - * [Option A - On a Linux container](#option-a---on-a-linux-container) + * [On a Linux container](#on-a-linux-container) * [Why containers?](#why-containers) * [Prerequisites](#prerequisites) * [Setup](#setup) @@ -21,12 +21,12 @@ You'll need to create an account but it's quick and free. # Step 2 - Installation -## Option A - On a Linux container +## On a Linux container ### Why containers? They provide a flexible, reliable way to build and deploy Stable Diffusion. We also use a Docker volume to store the largest model file and image outputs as a first step in decoupling storage and compute. Future enhancements will do this for other model files and assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important. -This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. +This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. You'll also need to specify the Stable Diffusion requirements file that matches your OS and architecture e.g. Linux on an arm64 chip if running a Linux container on Apple silicon. The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to an environment with NVIDIA GPUs on-premises or in the cloud. From 0aa3dfbc352cd0e3cfcb667ae77414dfb95f7011 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Mon, 12 Sep 2022 23:46:45 -0500 Subject: [PATCH 039/238] Update link to guide on Docker, supported architectures, and platform specifiers. --- docs/installation/INSTALL_DOCKER.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/INSTALL_DOCKER.md b/docs/installation/INSTALL_DOCKER.md index 86ba6dfb65..9a124b4e52 100644 --- a/docs/installation/INSTALL_DOCKER.md +++ b/docs/installation/INSTALL_DOCKER.md @@ -31,7 +31,7 @@ This example uses a Mac M1/M2 (arm64) but you can specify the platform and archi The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to an environment with NVIDIA GPUs on-premises or in the cloud. ### Prerequisites -[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#docker) +[Install Docker](https://github.com/santisbon/guides#docker) On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. Create a Docker volume for the downloaded model file From e1a6d0c13801269be99dbc731c0b02939a007e22 Mon Sep 17 00:00:00 2001 From: jspraul Date: Tue, 13 Sep 2022 08:29:14 -0400 Subject: [PATCH 040/238] web server does not supply embiggen options yet (#535) --- ldm/dream/pngwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index 4e8bca2366..9ac8a3160f 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -73,9 +73,9 @@ class PromptFormatter: switches.append(f'-G{opt.gfpgan_strength}') if opt.upscale: switches.append(f'-U {" ".join([str(u) for u in opt.upscale])}') - if opt.embiggen: + if hasattr(opt, 'embiggen') and opt.embiggen: switches.append(f'-embiggen {" ".join([str(u) for u in opt.embiggen])}') - if opt.embiggen_tiles: + if hasattr(opt, 'embiggen_tiles') and opt.embiggen_tiles: switches.append(f'-embiggen_tiles {" ".join([str(u) for u in opt.embiggen_tiles])}') if opt.variation_amount > 0: switches.append(f'-v{opt.variation_amount}') From d0a71dc3615036b2041fbeb31107f2c8364a8d36 Mon Sep 17 00:00:00 2001 From: Any-Winter-4079 <50542132+Any-Winter-4079@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:53:45 +0200 Subject: [PATCH 041/238] Update attention.py for 16-32GB M1 performance (#540) Code cleanup and attention.py einsum_ops update for M1 16-32GB performance. Expected: On par with fastest ever from 8 to 128GB for 512x512. Allows large images. --- ldm/modules/attention.py | 78 ++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 894c4db839..55e5b9f8a4 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -167,30 +167,25 @@ class CrossAttention(nn.Module): nn.Linear(inner_dim, query_dim), nn.Dropout(dropout) ) - - if not torch.cuda.is_available(): - mem_av = psutil.virtual_memory().available / (1024**3) - if mem_av > 32: - self.einsum_op = self.einsum_op_v1 - elif mem_av > 12: - self.einsum_op = self.einsum_op_v2 - else: - self.einsum_op = self.einsum_op_v3 - del mem_av + + if torch.cuda.is_available(): + self.einsum_op = self.einsum_op_cuda else: - self.einsum_op = self.einsum_op_v4 + self.mem_total = psutil.virtual_memory().total / (1024**3) + self.einsum_op = self.einsum_op_mps_v1 if self.mem_total >= 32 else self.einsum_op_mps_v2 - # mps 64-128 GB - def einsum_op_v1(self, q, k, v, r1): - if q.shape[1] <= 4096: # for 512x512: the max q.shape[1] is 4096 - s1 = einsum('b i d, b j d -> b i j', q, k) * self.scale # aggressive/faster: operation in one go - s2 = s1.softmax(dim=-1, dtype=q.dtype) - del s1 - r1 = einsum('b i j, b j d -> b i d', s2, v) - del s2 + def einsum_op_compvis(self, q, k, v, r1): + s1 = einsum('b i d, b j d -> b i j', q, k) * self.scale # faster + s2 = s1.softmax(dim=-1, dtype=q.dtype) + del s1 + r1 = einsum('b i j, b j d -> b i d', s2, v) + del s2 + return r1 + + def einsum_op_mps_v1(self, q, k, v, r1): + if q.shape[1] <= 4096: # (512x512) max q.shape[1]: 4096 + r1 = self.einsum_op_compvis(q, k, v, r1) else: - # q.shape[0] * q.shape[1] * slice_size >= 2**31 throws err - # needs around half of that slice_size to not generate noise slice_size = math.floor(2**30 / (q.shape[0] * q.shape[1])) for i in range(0, q.shape[1], slice_size): end = i + slice_size @@ -201,33 +196,22 @@ class CrossAttention(nn.Module): del s2 return r1 - # mps 16-32 GB (can be optimized) - def einsum_op_v2(self, q, k, v, r1): - slice_size = math.floor(2**30 / (q.shape[0] * q.shape[1])) - for i in range(0, q.shape[1], slice_size): # conservative/less mem: operation in steps - end = i + slice_size - s1 = einsum('b i d, b j d -> b i j', q[:, i:end], k) * self.scale - s2 = s1.softmax(dim=-1, dtype=r1.dtype) - del s1 - r1[:, i:end] = einsum('b i j, b j d -> b i d', s2, v) - del s2 + def einsum_op_mps_v2(self, q, k, v, r1): + if self.mem_total >= 8 and q.shape[1] <= 4096: + r1 = self.einsum_op_compvis(q, k, v, r1) + else: + slice_size = 1 + for i in range(0, q.shape[0], slice_size): + end = min(q.shape[0], i + slice_size) + s1 = einsum('b i d, b j d -> b i j', q[i:end], k[i:end]) + s1 *= self.scale + s2 = s1.softmax(dim=-1, dtype=r1.dtype) + del s1 + r1[i:end] = einsum('b i j, b j d -> b i d', s2, v[i:end]) + del s2 return r1 - - # mps 8 GB - def einsum_op_v3(self, q, k, v, r1): - slice_size = 1 - for i in range(0, q.shape[0], slice_size): # iterate over q.shape[0] - end = min(q.shape[0], i + slice_size) - s1 = einsum('b i d, b j d -> b i j', q[i:end], k[i:end]) # adapted einsum for mem - s1 *= self.scale - s2 = s1.softmax(dim=-1, dtype=r1.dtype) - del s1 - r1[i:end] = einsum('b i j, b j d -> b i d', s2, v[i:end]) # adapted einsum for mem - del s2 - return r1 - - # cuda - def einsum_op_v4(self, q, k, v, r1): + + def einsum_op_cuda(self, q, k, v, r1): stats = torch.cuda.memory_stats(q.device) mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] From ecb84ecc10423391b709d248153e6b8013be5153 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 13 Sep 2022 11:38:43 -0400 Subject: [PATCH 042/238] Add missing contributors. * A more systematic review of contributors needed. --- README.md | 5 ++++- docs/CONTRIBUTORS.md | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 539715fae8..e1978a4c1a 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,10 @@ A full set of contribution guidelines, along with templates, are in progress, bu ## **Contributors** -This fork is a combined effort of various people from across the world. [Check out the list of all these amazing people](docs/CONTRIBUTORS.md). We thank them for their time, hard work and effort. +This fork is a combined effort of various people from across the +world. [Check out the list of all these amazing +people](docs/CONTRIBUTORS.md). We thank them for their time, hard work +and effort. # Support diff --git a/docs/CONTRIBUTORS.md b/docs/CONTRIBUTORS.md index 57a9d5cd38..cca8bade9b 100644 --- a/docs/CONTRIBUTORS.md +++ b/docs/CONTRIBUTORS.md @@ -45,6 +45,9 @@ _Contributions by:_ - [Paul Sajna](https://github.com/sajattack) - [Samuel Husso](https://github.com/shusso) - [nicolai256](https://github.com/nicolai256) +- [Mihai](https://github.com/mh-dm) +- [Any Winter](https://github.com/any-winter-4079) +- [Doggettx](https://github.com/doggettx) _Original CompVis Authors:_ From d15c75ecae6a38544f952a4d3a384c08fc51539b Mon Sep 17 00:00:00 2001 From: tildebyte <337875+tildebyte@users.noreply.github.com> Date: Wed, 14 Sep 2022 07:01:58 -0400 Subject: [PATCH 043/238] TOIL(pip): Refactor pip requirements across the board (#531) * Refactor pip requirements across the board Signed-off-by: Ben Alkov * fix name, version in setup.py Signed-off-by: Ben Alkov * Update notebooks for new requirements file changes Signed-off-by: Ben Alkov --- .../Stable-Diffusion-local-Windows.ipynb | 68 +---- notebooks/Stable_Diffusion_AI_Notebook.ipynb | 268 +++++++++--------- requirements-colab.txt | 26 -- requirements-lin-AMD.txt | 7 + requirements-lin-win-colab-CUDA.txt | 7 + requirements-lin.txt | 33 --- requirements-mac-MPS-CPU.txt | 8 + requirements-mac.txt | 24 -- requirements-win.txt | 33 --- requirements.txt | 27 ++ setup.py | 4 +- 11 files changed, 191 insertions(+), 314 deletions(-) delete mode 100644 requirements-colab.txt create mode 100644 requirements-lin-AMD.txt create mode 100644 requirements-lin-win-colab-CUDA.txt delete mode 100644 requirements-lin.txt create mode 100644 requirements-mac-MPS-CPU.txt delete mode 100644 requirements-mac.txt delete mode 100644 requirements-win.txt create mode 100644 requirements.txt diff --git a/notebooks/Stable-Diffusion-local-Windows.ipynb b/notebooks/Stable-Diffusion-local-Windows.ipynb index 1c5e90dcad..45495d6d40 100644 --- a/notebooks/Stable-Diffusion-local-Windows.ipynb +++ b/notebooks/Stable-Diffusion-local-Windows.ipynb @@ -40,57 +40,9 @@ "outputs": [], "source": [ "%%cmd\n", - "git clone https://github.com/lstein/stable-diffusion.git" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%cd stable-diffusion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile requirements.txt\n", - "albumentations==0.4.3\n", - "einops==0.3.0\n", - "huggingface-hub==0.8.1\n", - "imageio-ffmpeg==0.4.2\n", - "imageio==2.9.0\n", - "kornia==0.6.0\n", - "# pip will resolve the version which matches torch\n", - "numpy\n", - "omegaconf==2.1.1\n", - "opencv-python==4.6.0.66\n", - "pillow==9.2.0\n", - "pip>=22\n", - "pudb==2019.2\n", - "pytorch-lightning==1.4.2\n", - "streamlit==1.12.0\n", - "# \"CompVis/taming-transformers\" doesn't work\n", - "# ldm\\models\\autoencoder.py\", line 6, in \n", - "# from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer\n", - "# ModuleNotFoundError\n", - "taming-transformers-rom1504==0.0.6\n", - "test-tube>=0.7.5\n", - "torch-fidelity==0.3.0\n", - "torchmetrics==0.6.0\n", - "transformers==4.19.2\n", - "git+https://github.com/openai/CLIP.git@main#egg=clip\n", - "git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion\n", - "# No CUDA in PyPi builds\n", - "--extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org\n", - "torch==1.11.0\n", - "# Same as numpy - let pip do its thing\n", - "torchvision\n", - "-e .\n" + "git clone https://github.com/lstein/stable-diffusion.git\n", + "cd /content/stable-diffusion/\n", + "git checkout --quiet development" ] }, { @@ -100,14 +52,14 @@ "outputs": [], "source": [ "%%cmd\n", - "pew new --python 3.10 -r requirements.txt --dont-activate ldm" + "pew new --python 3.10 -r requirements-lin-win-colab-CUDA.txt --dont-activate stable-diffusion" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Switch the notebook kernel to the new 'ldm' environment!\n", + "# Switch the notebook kernel to the new 'stable-diffusion' environment!\n", "\n", "## VSCode: restart VSCode and come back to this cell\n", "\n", @@ -115,7 +67,7 @@ "1. Type \"Select Interpreter\" and select \"Jupyter: Select Interpreter to Start Jupyter Server\"\n", "1. VSCode will say that it needs to install packages. Click the \"Install\" button.\n", "1. Once the install is finished, do 1 & 2 again\n", - "1. Pick 'ldm'\n", + "1. Pick 'stable-diffusion'\n", "1. Run the following cell" ] }, @@ -136,7 +88,7 @@ "## Jupyter/JupyterLab\n", "\n", "1. Run the cell below\n", - "1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'ldm' from the drop-down.\n" + "1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'stable-diffusion' from the drop-down.\n" ] }, { @@ -154,9 +106,9 @@ "source": [ "# DO NOT RUN THIS CELL IF YOU ARE USING VSCODE!!\n", "%%cmd\n", - "pew workon ldm\n", + "pew workon stable-diffusion\n", "pip3 install ipykernel\n", - "python -m ipykernel install --name=ldm" + "python -m ipykernel install --name=stable-diffusion" ] }, { @@ -231,7 +183,7 @@ "Now:\n", "\n", "1. `cd` to wherever the 'stable-diffusion' directory is\n", - "1. Run `pew workon ldm`\n", + "1. Run `pew workon stable-diffusion`\n", "1. Run `winpty python scripts\\dream.py`" ] } diff --git a/notebooks/Stable_Diffusion_AI_Notebook.ipynb b/notebooks/Stable_Diffusion_AI_Notebook.ipynb index 3508a62efa..129323bb15 100644 --- a/notebooks/Stable_Diffusion_AI_Notebook.ipynb +++ b/notebooks/Stable_Diffusion_AI_Notebook.ipynb @@ -1,27 +1,12 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "collapsed_sections": [], - "private_outputs": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU", - "gpuClass": "standard" - }, "cells": [ { "cell_type": "markdown", + "metadata": { + "id": "ycYWcsEKc6w7" + }, "source": [ - "# Stable Diffusion AI Notebook (Release 1.13)\n", + "# Stable Diffusion AI Notebook (Release 1.14)\n", "\n", "\"stable-diffusion-ai\"
\n", "#### Instructions:\n", @@ -35,33 +20,30 @@ "Requirements: For this notebook to work you need to have [Stable-Diffusion-v-1-4](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) stored in your Google Drive, it will be needed in cell #7\n", "##### For more details visit Github repository: [lstein/stable-diffusion](https://github.com/lstein/stable-diffusion)\n", "---\n" - ], - "metadata": { - "id": "ycYWcsEKc6w7" - } + ] }, { "cell_type": "markdown", - "source": [ - "## ◢ Installation" - ], "metadata": { "id": "dr32VLxlnouf" - } + }, + "source": [ + "## ◢ Installation" + ] }, { "cell_type": "code", - "source": [ - "#@title 1. Check current GPU assigned\n", - "!nvidia-smi -L\n", - "!nvidia-smi" - ], + "execution_count": null, "metadata": { "cellView": "form", "id": "a2Z5Qu_o8VtQ" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "#@title 1. Check current GPU assigned\n", + "!nvidia-smi -L\n", + "!nvidia-smi" + ] }, { "cell_type": "code", @@ -75,90 +57,91 @@ "#@title 2. Download stable-diffusion Repository\n", "from os.path import exists\n", "\n", - "if exists(\"/content/stable-diffusion/\")==True:\n", - " %cd /content/stable-diffusion/\n", - " print(\"Already downloaded repo\")\n", - "else:\n", - " !git clone --quiet https://github.com/lstein/stable-diffusion.git # Original repo\n", - " %cd /content/stable-diffusion/\n", - " !git checkout --quiet tags/release-1.13" + "!git clone --quiet https://github.com/lstein/stable-diffusion.git # Original repo\n", + "%cd /content/stable-diffusion/\n", + "!git checkout --quiet tags/release-1.14.1" ] }, { "cell_type": "code", - "source": [ - "#@title 3. Install dependencies\n", - "import gc\n", - "\n", - "if exists(\"/content/stable-diffusion/requirements-colab.txt\")==True:\n", - " %cd /content/stable-diffusion/\n", - " print(\"Already downloaded requirements file\")\n", - "else:\n", - " !wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements-colab.txt\n", - "!pip install colab-xterm\n", - "!pip install -r requirements-colab.txt\n", - "gc.collect()" - ], + "execution_count": null, "metadata": { "cellView": "form", "id": "QbXcGXYEFSNB" }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", + "outputs": [], "source": [ - "#@title 4. Load small ML models required\n", - "%cd /content/stable-diffusion/\n", - "!python scripts/preload_models.py\n", + "#@title 3. Install dependencies\n", + "import gc\n", + "\n", + "!wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements.txt\n", + "!wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements-lin-win-colab-CUDA.txt\n", + "!pip install colab-xterm\n", + "!pip install -r requirements-lin-win-colab-CUDA.txt\n", + "!pip install clean-fid torchtext\n", "gc.collect()" - ], - "metadata": { - "cellView": "form", - "id": "ChIDWxLVHGGJ" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", - "source": [ - "#@title 5. Restart Runtime\n", - "exit()" - ], + "execution_count": null, "metadata": { "cellView": "form", "id": "8rSMhgnAttQa" }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", + "outputs": [], "source": [ - "## ◢ Configuration" - ], - "metadata": { - "id": "795x1tMoo8b1" - } + "#@title 4. Restart Runtime\n", + "exit()" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ChIDWxLVHGGJ" + }, + "outputs": [], "source": [ - "#@title 6. Mount google Drive\n", - "from google.colab import drive\n", - "drive.mount('/content/drive')" - ], + "#@title 5. Load small ML models required\n", + "import gc\n", + "%cd /content/stable-diffusion/\n", + "!python scripts/preload_models.py\n", + "gc.collect()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "795x1tMoo8b1" + }, + "source": [ + "## ◢ Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", "id": "YEWPV-sF1RDM" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "#@title 6. Mount google Drive\n", + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "zRTJeZ461WGu" + }, + "outputs": [], "source": [ "#@title 7. Drive Path to model\n", "#@markdown Path should start with /content/drive/path-to-your-file
\n", @@ -167,20 +150,20 @@ "from os.path import exists\n", "\n", "model_path = \"\" #@param {type:\"string\"}\n", - "if exists(model_path)==True:\n", - " print(\"✅ Valid directory\")\n", + "if exists(model_path):\n", + " print(\"✅ Valid directory\")\n", "else: \n", - " print(\"❌ File doesn't exist\")" - ], - "metadata": { - "cellView": "form", - "id": "zRTJeZ461WGu" - }, - "execution_count": null, - "outputs": [] + " print(\"❌ File doesn't exist\")" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "UY-NNz4I8_aG" + }, + "outputs": [], "source": [ "#@title 8. Symlink to model\n", "\n", @@ -188,39 +171,39 @@ "import os \n", "\n", "# Folder creation if it doesn't exist\n", - "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1\")==True:\n", - " print(\"❗ Dir stable-diffusion-v1 already exists\")\n", + "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1\"):\n", + " print(\"❗ Dir stable-diffusion-v1 already exists\")\n", "else:\n", - " %mkdir /content/stable-diffusion/models/ldm/stable-diffusion-v1\n", - " print(\"✅ Dir stable-diffusion-v1 created\")\n", + " %mkdir /content/stable-diffusion/models/ldm/stable-diffusion-v1\n", + " print(\"✅ Dir stable-diffusion-v1 created\")\n", "\n", "# Symbolic link if it doesn't exist\n", - "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt\")==True:\n", - " print(\"❗ Symlink already created\")\n", + "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt\"):\n", + " print(\"❗ Symlink already created\")\n", "else: \n", - " src = model_path\n", - " dst = '/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt'\n", - " os.symlink(src, dst) \n", - " print(\"✅ Symbolic link created successfully\")" - ], - "metadata": { - "id": "UY-NNz4I8_aG", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + " src = model_path\n", + " dst = '/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt'\n", + " os.symlink(src, dst) \n", + " print(\"✅ Symbolic link created successfully\")" + ] }, { "cell_type": "markdown", - "source": [ - "## ◢ Execution" - ], "metadata": { "id": "Mc28N0_NrCQH" - } + }, + "source": [ + "## ◢ Execution" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ir4hCrMIuUpl" + }, + "outputs": [], "source": [ "#@title 9. Run Terminal and Execute Dream bot\n", "#@markdown Steps:
\n", @@ -229,24 +212,21 @@ "#@markdown 3. Example text: `Astronaut floating in a distant galaxy`
\n", "#@markdown 4. To quit Dream bot use: `q` command.
\n", "\n", - "import gc\n", - "%cd /content/stable-diffusion/\n", "%load_ext colabxterm\n", "%xterm\n", "gc.collect()" - ], - "metadata": { - "id": "ir4hCrMIuUpl", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "qnLohSHmKoGk" + }, + "outputs": [], "source": [ "#@title 10. Show the last 15 generated images\n", - "import gc\n", "import glob\n", "import matplotlib.pyplot as plt\n", "import matplotlib.image as mpimg\n", @@ -269,13 +249,25 @@ " plt.imshow(image)\n", " gc.collect()\n", "\n" - ], - "metadata": { - "cellView": "form", - "id": "qnLohSHmKoGk" - }, - "execution_count": null, - "outputs": [] + ] } - ] + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "private_outputs": true, + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/requirements-colab.txt b/requirements-colab.txt deleted file mode 100644 index f9cc5600ea..0000000000 --- a/requirements-colab.txt +++ /dev/null @@ -1,26 +0,0 @@ -albumentations==0.4.3 -clean-fid==0.1.29 -einops==0.3.0 -huggingface-hub==0.8.1 -imageio-ffmpeg==0.4.2 -imageio==2.9.0 -kornia==0.6.0 -numpy==1.21.6 -omegaconf==2.1.1 -opencv-python==4.6.0.66 -pillow==9.2.0 -pip>=22 -pudb==2019.2 -pytorch-lightning==1.4.2 -streamlit==1.12.0 -taming-transformers-rom1504==0.0.6 -test-tube>=0.7.5 -torch-fidelity==0.3.0 -torchmetrics==0.6.0 -torchtext==0.6.0 -transformers==4.19.2 -torch==1.12.1+cu113 -torchvision==0.13.1+cu113 -git+https://github.com/openai/CLIP.git@main#egg=clip -git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion --e . diff --git a/requirements-lin-AMD.txt b/requirements-lin-AMD.txt new file mode 100644 index 0000000000..d12996b45a --- /dev/null +++ b/requirements-lin-AMD.txt @@ -0,0 +1,7 @@ +-r requirements.txt + +# Get hardware-appropriate torch/torchvision +--extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org +torch +torchvision +-e . diff --git a/requirements-lin-win-colab-CUDA.txt b/requirements-lin-win-colab-CUDA.txt new file mode 100644 index 0000000000..91c8d0f35c --- /dev/null +++ b/requirements-lin-win-colab-CUDA.txt @@ -0,0 +1,7 @@ +-r requirements.txt + +# Get hardware-appropriate torch/torchvision +--extra-index-url https://download.pytorch.org/whl/cu116 --trusted-host https://download.pytorch.org +torch +torchvision +-e . diff --git a/requirements-lin.txt b/requirements-lin.txt deleted file mode 100644 index 3f22122eec..0000000000 --- a/requirements-lin.txt +++ /dev/null @@ -1,33 +0,0 @@ -albumentations==0.4.3 -einops==0.3.0 -huggingface-hub==0.8.1 -imageio-ffmpeg==0.4.2 -imageio==2.9.0 -kornia==0.6.0 -# pip will resolve the version which matches torch -numpy -omegaconf==2.1.1 -opencv-python==4.6.0.66 -pillow==9.2.0 -pip>=22 -pudb==2019.2 -pytorch-lightning==1.4.2 -streamlit==1.12.0 -# "CompVis/taming-transformers" doesn't work -# ldm\models\autoencoder.py", line 6, in -# from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer -# ModuleNotFoundError -taming-transformers-rom1504==0.0.6 -test-tube>=0.7.5 -torch-fidelity==0.3.0 -torchmetrics==0.6.0 -transformers==4.19.2 -git+https://github.com/openai/CLIP.git@main#egg=clip -git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion -git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan -# No CUDA in PyPi builds ---extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org -torch==1.11.0 -# Same as numpy - let pip do its thing -torchvision --e . diff --git a/requirements-mac-MPS-CPU.txt b/requirements-mac-MPS-CPU.txt new file mode 100644 index 0000000000..b6b1f2bb2f --- /dev/null +++ b/requirements-mac-MPS-CPU.txt @@ -0,0 +1,8 @@ +-r requirements.txt + +--pre +--extra-index-url https://download.pytorch.org/whl/nightly/cpu --trusted-host https://download.pytorch.org + +torch +torchvision +-e . diff --git a/requirements-mac.txt b/requirements-mac.txt deleted file mode 100644 index 7296c84cc5..0000000000 --- a/requirements-mac.txt +++ /dev/null @@ -1,24 +0,0 @@ -albumentations==0.4.3 -einops==0.3.0 -huggingface-hub==0.8.1 -imageio==2.9.0 -imageio-ffmpeg==0.4.2 -kornia==0.6.0 -numpy==1.23.1 ---pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu -omegaconf==2.1.1 -opencv-python==4.6.0.66 -pillow==9.2.0 -pudb==2019.2 -torch==1.12.1 -torchvision==0.13.0 -pytorch-lightning==1.4.2 -streamlit==1.12.0 -test-tube>=0.7.5 -torch-fidelity==0.3.0 -torchmetrics==0.6.0 -transformers==4.19.2 --e git+https://github.com/openai/CLIP.git@main#egg=clip --e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers --e git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion --e git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan diff --git a/requirements-win.txt b/requirements-win.txt deleted file mode 100644 index 3f22122eec..0000000000 --- a/requirements-win.txt +++ /dev/null @@ -1,33 +0,0 @@ -albumentations==0.4.3 -einops==0.3.0 -huggingface-hub==0.8.1 -imageio-ffmpeg==0.4.2 -imageio==2.9.0 -kornia==0.6.0 -# pip will resolve the version which matches torch -numpy -omegaconf==2.1.1 -opencv-python==4.6.0.66 -pillow==9.2.0 -pip>=22 -pudb==2019.2 -pytorch-lightning==1.4.2 -streamlit==1.12.0 -# "CompVis/taming-transformers" doesn't work -# ldm\models\autoencoder.py", line 6, in -# from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer -# ModuleNotFoundError -taming-transformers-rom1504==0.0.6 -test-tube>=0.7.5 -torch-fidelity==0.3.0 -torchmetrics==0.6.0 -transformers==4.19.2 -git+https://github.com/openai/CLIP.git@main#egg=clip -git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion -git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan -# No CUDA in PyPi builds ---extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org -torch==1.11.0 -# Same as numpy - let pip do its thing -torchvision --e . diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..2007ca4caf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +--prefer-binary + +albumentations +einops +huggingface-hub +imageio-ffmpeg +imageio +kornia +# pip will resolve the version which matches torch +numpy +omegaconf +opencv-python +pillow +pip>=22 +pudb +pytorch-lightning +streamlit +# "CompVis/taming-transformers" IS NOT INSTALLABLE +# This is a drop-in replacement +taming-transformers-rom1504 +test-tube +torch-fidelity +torchmetrics +transformers +git+https://github.com/openai/CLIP.git@main#egg=clip +git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion +git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan diff --git a/setup.py b/setup.py index a24d541676..3baab3abc4 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from setuptools import setup, find_packages setup( - name='latent-diffusion', - version='0.0.1', + name='stable-diffusion', + version='1.15.0-dev', description='', packages=find_packages(), install_requires=[ From e6179af46a75afb5118ba1981da0e6594d910803 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 14 Sep 2022 07:02:31 -0400 Subject: [PATCH 044/238] Refactor generate.py and dream.py (#534) * revert inadvertent change of conda env name (#528) * Refactor generate.py and dream.py * config file path (models.yaml) is parsed inside Generate() to simplify API * Better handling of keyboard interrupts in file loading mode vs interactive * Removed oodles of unused variables. * move nonfunctional inpainting out of the scripts directory * fix ugly ddim tqdm formatting --- ldm/generate.py | 204 +++++++++++++------------- ldm/models/diffusion/ddim.py | 2 +- scripts/dream.py | 97 +++++------- scripts/{ => orig_scripts}/inpaint.py | 0 4 files changed, 141 insertions(+), 162 deletions(-) rename scripts/{ => orig_scripts}/inpaint.py (100%) diff --git a/ldm/generate.py b/ldm/generate.py index 52c8846d80..1bb8e33eb9 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -17,7 +17,7 @@ import transformers from omegaconf import OmegaConf from PIL import Image, ImageOps from torch import nn -from pytorch_lightning import seed_everything +from pytorch_lightning import seed_everything, logging from ldm.util import instantiate_from_config from ldm.models.diffusion.ddim import DDIMSampler @@ -35,7 +35,7 @@ Example Usage: from ldm.generate import Generate # Create an object with default values -gr = Generate() +gr = Generate('stable-diffusion-1.4') # do the slow model initialization gr.load_model() @@ -79,16 +79,17 @@ still work. The full list of arguments to Generate() are: gr = Generate( + # these values are set once and shouldn't be changed + conf = path to configuration file ('configs/models.yaml') + model = symbolic name of the model in the configuration file + full_precision = False + + # this value is sticky and maintained between generation calls + sampler_name = ['ddim', 'k_dpm_2_a', 'k_dpm_2', 'k_euler_a', 'k_euler', 'k_heun', 'k_lms', 'plms'] // k_lms + + # these are deprecated - use conf and model instead weights = path to model weights ('models/ldm/stable-diffusion-v1/model.ckpt') - config = path to model configuraiton ('configs/stable-diffusion/v1-inference.yaml') - iterations = // how many times to run the sampling (1) - steps = // 50 - seed = // current system time - sampler_name= ['ddim', 'k_dpm_2_a', 'k_dpm_2', 'k_euler_a', 'k_euler', 'k_heun', 'k_lms', 'plms'] // k_lms - grid = // false - width = // image width, multiple of 64 (512) - height = // image height, multiple of 64 (512) - cfg_scale = // condition-free guidance scale (7.5) + config = path to model configuraiton ('configs/stable-diffusion/v1-inference.yaml') ) """ @@ -101,66 +102,62 @@ class Generate: def __init__( self, - iterations = 1, - steps = 50, - cfg_scale = 7.5, - weights = 'models/ldm/stable-diffusion-v1/model.ckpt', - config = 'configs/stable-diffusion/v1-inference.yaml', - grid = False, - width = 512, - height = 512, + model = 'stable-diffusion-1.4', + conf = 'configs/models.yaml', + embedding_path = None, sampler_name = 'k_lms', ddim_eta = 0.0, # deterministic full_precision = False, - strength = 0.75, # default in scripts/img2img.py - seamless = False, - embedding_path = None, - device_type = 'cuda', - ignore_ctrl_c = False, + # these are deprecated; if present they override values in the conf file + weights = None, + config = None, ): - self.iterations = iterations - self.width = width - self.height = height - self.steps = steps - self.cfg_scale = cfg_scale - self.weights = weights - self.config = config - self.sampler_name = sampler_name - self.grid = grid - self.ddim_eta = ddim_eta - self.full_precision = True if choose_torch_device() == 'mps' else full_precision - self.strength = strength - self.seamless = seamless - self.embedding_path = embedding_path - self.device_type = device_type - self.ignore_ctrl_c = ignore_ctrl_c # note, this logic probably doesn't belong here... - self.model = None # empty for now - self.sampler = None - self.device = None - self.generators = {} - self.base_generator = None - self.seed = None + models = OmegaConf.load(conf) + mconfig = models[model] + self.weights = mconfig.weights if weights is None else weights + self.config = mconfig.config if config is None else config + self.height = mconfig.height + self.width = mconfig.width + self.iterations = 1 + self.steps = 50 + self.cfg_scale = 7.5 + self.sampler_name = sampler_name + self.ddim_eta = 0.0 # same seed always produces same image + self.full_precision = True if choose_torch_device() == 'mps' else full_precision + self.strength = 0.75 + self.seamless = False + self.embedding_path = embedding_path + self.model = None # empty for now + self.sampler = None + self.device = None + self.session_peakmem = None + self.generators = {} + self.base_generator = None + self.seed = None - if device_type == 'cuda' and not torch.cuda.is_available(): - device_type = choose_torch_device() - print(">> cuda not available, using device", device_type) + # Note that in previous versions, there was an option to pass the + # device to Generate(). However the device was then ignored, so + # it wasn't actually doing anything. This logic could be reinstated. + device_type = choose_torch_device() self.device = torch.device(device_type) # for VRAM usage statistics - device_type = choose_torch_device() - self.session_peakmem = torch.cuda.max_memory_allocated() if device_type == 'cuda' else None + self.session_peakmem = torch.cuda.max_memory_allocated() if self._has_cuda else None transformers.logging.set_verbosity_error() + # gets rid of annoying messages about random seed + logging.getLogger('pytorch_lightning').setLevel(logging.ERROR) + def prompt2png(self, prompt, outdir, **kwargs): """ Takes a prompt and an output directory, writes out the requested number of PNG files, and returns an array of [[filename,seed],[filename,seed]...] Optional named arguments are the same as those passed to Generate and prompt2image() """ - results = self.prompt2image(prompt, **kwargs) + results = self.prompt2image(prompt, **kwargs) pngwriter = PngWriter(outdir) - prefix = pngwriter.unique_prefix() - outputs = [] + prefix = pngwriter.unique_prefix() + outputs = [] for image, seed in results: name = f'{prefix}.{seed}.png' path = pngwriter.save_image_and_prompt_to_png( @@ -183,33 +180,35 @@ class Generate: self, # these are common prompt, - iterations = None, - steps = None, - seed = None, - cfg_scale = None, - ddim_eta = None, - skip_normalize = False, - image_callback = None, - step_callback = None, - width = None, - height = None, - sampler_name = None, - seamless = False, - log_tokenization= False, - with_variations = None, - variation_amount = 0.0, + iterations = None, + steps = None, + seed = None, + cfg_scale = None, + ddim_eta = None, + skip_normalize = False, + image_callback = None, + step_callback = None, + width = None, + height = None, + sampler_name = None, + seamless = False, + log_tokenization = False, + with_variations = None, + variation_amount = 0.0, # these are specific to img2img and inpaint - init_img = None, - init_mask = None, - fit = False, - strength = None, + init_img = None, + init_mask = None, + fit = False, + strength = None, # these are specific to embiggen (which also relies on img2img args) embiggen = None, embiggen_tiles = None, # these are specific to GFPGAN/ESRGAN - gfpgan_strength= 0, - save_original = False, - upscale = None, + gfpgan_strength = 0, + save_original = False, + upscale = None, + # Set this True to handle KeyboardInterrupt internally + catch_interrupts = False, **args, ): # eat up additional cruft """ @@ -262,10 +261,9 @@ class Generate: self.log_tokenization = log_tokenization with_variations = [] if with_variations is None else with_variations - model = ( - self.load_model() - ) # will instantiate the model or return it from cache - + # will instantiate the model or return it from cache + model = self.load_model() + for m in model.modules(): if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): m.padding_mode = 'circular' if seamless else m._orig_padding_mode @@ -281,7 +279,6 @@ class Generate: (embiggen == None and embiggen_tiles == None) or ((embiggen != None or embiggen_tiles != None) and init_img != None) ), 'Embiggen requires an init/input image to be specified' - # check this logic - doesn't look right if len(with_variations) > 0 or variation_amount > 1.0: assert seed is not None,\ 'seed must be specified when using with_variations' @@ -298,7 +295,7 @@ class Generate: self._set_sampler() tic = time.time() - if torch.cuda.is_available(): + if self._has_cuda(): torch.cuda.reset_peak_memory_stats() results = list() @@ -307,9 +304,9 @@ class Generate: try: uc, c = get_uc_and_c( - prompt, model=self.model, + prompt, model =self.model, skip_normalize=skip_normalize, - log_tokens=self.log_tokenization + log_tokens =self.log_tokenization ) (init_image,mask_image) = self._make_images(init_img,init_mask, width, height, fit) @@ -352,27 +349,25 @@ class Generate: save_original = save_original, image_callback = image_callback) - except KeyboardInterrupt: - print('*interrupted*') - if not self.ignore_ctrl_c: - raise KeyboardInterrupt - print( - '>> Partial results will be returned; if --grid was requested, nothing will be returned.' - ) except RuntimeError as e: print(traceback.format_exc(), file=sys.stderr) print('>> Could not generate image.') + except KeyboardInterrupt: + if catch_interrupts: + print('**Interrupted** Partial results will be returned.') + else: + raise KeyboardInterrupt toc = time.time() print('>> Usage stats:') print( f'>> {len(results)} image(s) generated in', '%4.2fs' % (toc - tic) ) - if torch.cuda.is_available() and self.device.type == 'cuda': + if self._has_cuda(): print( f'>> Max VRAM used for this generation:', '%4.2fG.' % (torch.cuda.max_memory_allocated() / 1e9), - 'Current VRAM utilization:' + 'Current VRAM utilization:', '%4.2fG' % (torch.cuda.memory_allocated() / 1e9), ) @@ -439,8 +434,7 @@ class Generate: if self.model is None: seed_everything(random.randrange(0, np.iinfo(np.uint32).max)) try: - config = OmegaConf.load(self.config) - model = self._load_model_from_config(config, self.weights) + model = self._load_model_from_config(self.config, self.weights) if self.embedding_path is not None: model.embedding_manager.load( self.embedding_path, self.full_precision @@ -541,8 +535,11 @@ class Generate: print(msg) - def _load_model_from_config(self, config, ckpt): - print(f'>> Loading model from {ckpt}') + # Be warned: config is the path to the model config file, not the dream conf file! + # Also note that we can get config and weights from self, so why do we need to + # pass them as args? + def _load_model_from_config(self, config, weights): + print(f'>> Loading model from {weights}') # for usage statistics device_type = choose_torch_device() @@ -551,10 +548,11 @@ class Generate: tic = time.time() # this does the work - pl_sd = torch.load(ckpt, map_location='cpu') - sd = pl_sd['state_dict'] - model = instantiate_from_config(config.model) - m, u = model.load_state_dict(sd, strict=False) + c = OmegaConf.load(config) + pl_sd = torch.load(weights, map_location='cpu') + sd = pl_sd['state_dict'] + model = instantiate_from_config(c.model) + m, u = model.load_state_dict(sd, strict=False) if self.full_precision: print( @@ -573,7 +571,7 @@ class Generate: print( f'>> Model loaded in', '%4.2fs' % (toc - tic) ) - if device_type == 'cuda': + if self._has_cuda(): print( '>> Max VRAM used to load the model:', '%4.2fG' % (torch.cuda.max_memory_allocated() / 1e9), @@ -710,3 +708,5 @@ class Generate: return width, height, resize_needed + def _has_cuda(self): + return self.device.type == 'cuda' diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index 3868540526..b875aac331 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -225,7 +225,7 @@ class DDIMSampler(object): total_steps = ( timesteps if ddim_use_original_steps else timesteps.shape[0] ) - print(f'Running DDIM Sampling with {total_steps} timesteps') + print(f'\nRunning DDIM Sampling with {total_steps} timesteps') iterator = tqdm( time_range, diff --git a/scripts/dream.py b/scripts/dream.py index 8559c1b083..aec27506c3 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -33,53 +33,35 @@ def main(): print('--weights argument has been deprecated. Please configure ./configs/models.yaml, and call it using --model instead.') sys.exit(-1) - try: - models = OmegaConf.load(opt.config) - width = models[opt.model].width - height = models[opt.model].height - config = models[opt.model].config - weights = models[opt.model].weights - except (FileNotFoundError, IOError, KeyError) as e: - print(f'{e}. Aborting.') - sys.exit(-1) - print('* Initializing, be patient...\n') sys.path.append('.') - from pytorch_lightning import logging from ldm.generate import Generate # these two lines prevent a horrible warning message from appearing # when the frozen CLIP tokenizer is imported import transformers - transformers.logging.set_verbosity_error() - # creating a simple text2image object with a handful of + # creating a simple Generate object with a handful of # defaults passed on the command line. # additional parameters will be added (or overriden) during # the user input loop - t2i = Generate( - width=width, - height=height, - sampler_name=opt.sampler_name, - weights=weights, - full_precision=opt.full_precision, - config=config, - grid=opt.grid, - # this is solely for recreating the prompt - seamless=opt.seamless, - embedding_path=opt.embedding_path, - device_type=opt.device, - ignore_ctrl_c=opt.infile is None, - ) + try: + gen = Generate( + conf = opt.config, + model = opt.model, + sampler_name = opt.sampler_name, + embedding_path = opt.embedding_path, + full_precision = opt.full_precision, + ) + except (FileNotFoundError, IOError, KeyError) as e: + print(f'{e}. Aborting.') + sys.exit(-1) # make sure the output directory exists if not os.path.exists(opt.outdir): os.makedirs(opt.outdir) - # gets rid of annoying messages about random seed - logging.getLogger('pytorch_lightning').setLevel(logging.ERROR) - # load the infile as a list of lines infile = None if opt.infile: @@ -98,21 +80,23 @@ def main(): print(">> changed to seamless tiling mode") # preload the model - t2i.load_model() + gen.load_model() if not infile: print( "\n* Initialization done! Awaiting your command (-h for help, 'q' to quit)" ) - cmd_parser = create_cmd_parser() + # web server loops forever if opt.web: - dream_server_loop(t2i, opt.host, opt.port, opt.outdir) - else: - main_loop(t2i, opt.outdir, opt.prompt_as_dir, cmd_parser, infile) + dream_server_loop(gen, opt.host, opt.port, opt.outdir) + sys.exit(0) + cmd_parser = create_cmd_parser() + main_loop(gen, opt.outdir, opt.prompt_as_dir, cmd_parser, infile) -def main_loop(t2i, outdir, prompt_as_dir, parser, infile): +# TODO: main_loop() has gotten busy. Needs to be refactored. +def main_loop(gen, outdir, prompt_as_dir, parser, infile): """prompt/read/execute loop""" done = False path_filter = re.compile(r'[<>:"/\\|?*]') @@ -132,9 +116,6 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): except EOFError: done = True continue - except KeyboardInterrupt: - done = True - continue # skip empty lines if not command.strip(): @@ -184,6 +165,7 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): if len(opt.prompt) == 0: print('Try again with a prompt!') continue + # retrieve previous value! if opt.init_img is not None and re.match('^-\\d+$', opt.init_img): try: @@ -204,8 +186,6 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): opt.seed = None continue - do_grid = opt.grid or t2i.grid - if opt.with_variations is not None: # shotgun parsing, woo parts = [] @@ -258,11 +238,11 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): file_writer = PngWriter(current_outdir) prefix = file_writer.unique_prefix() results = [] # list of filename, prompt pairs - grid_images = dict() # seed -> Image, only used if `do_grid` + grid_images = dict() # seed -> Image, only used if `opt.grid` def image_writer(image, seed, upscaled=False): path = None - if do_grid: + if opt.grid: grid_images[seed] = image else: if upscaled and opt.save_original: @@ -278,16 +258,16 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): iter_opt.with_variations = opt.with_variations + this_variation iter_opt.variation_amount = 0 normalized_prompt = PromptFormatter( - t2i, iter_opt).normalize_prompt() + gen, iter_opt).normalize_prompt() metadata_prompt = f'{normalized_prompt} -S{iter_opt.seed}' elif opt.with_variations is not None: normalized_prompt = PromptFormatter( - t2i, opt).normalize_prompt() + gen, opt).normalize_prompt() # use the original seed - the per-iteration value is the last variation-seed metadata_prompt = f'{normalized_prompt} -S{opt.seed}' else: normalized_prompt = PromptFormatter( - t2i, opt).normalize_prompt() + gen, opt).normalize_prompt() metadata_prompt = f'{normalized_prompt} -S{seed}' path = file_writer.save_image_and_prompt_to_png( image, metadata_prompt, filename) @@ -296,16 +276,21 @@ def main_loop(t2i, outdir, prompt_as_dir, parser, infile): results.append([path, metadata_prompt]) last_results.append([path, seed]) - t2i.prompt2image(image_callback=image_writer, **vars(opt)) + catch_ctrl_c = infile is None # if running interactively, we catch keyboard interrupts + gen.prompt2image( + image_callback=image_writer, + catch_interrupts=catch_ctrl_c, + **vars(opt) + ) - if do_grid and len(grid_images) > 0: + if opt.grid and len(grid_images) > 0: grid_img = make_grid(list(grid_images.values())) grid_seeds = list(grid_images.keys()) first_seed = last_results[0][1] filename = f'{prefix}.{first_seed}.png' # TODO better metadata for grid images normalized_prompt = PromptFormatter( - t2i, opt).normalize_prompt() + gen, opt).normalize_prompt() metadata_prompt = f'{normalized_prompt} -S{first_seed} --grid -n{len(grid_images)} # {grid_seeds}' path = file_writer.save_image_and_prompt_to_png( grid_img, metadata_prompt, filename @@ -337,11 +322,12 @@ def get_next_command(infile=None) -> str: # command string raise EOFError else: command = command.strip() - print(f'#{command}') + if len(command)>0: + print(f'#{command}') return command -def dream_server_loop(t2i, host, port, outdir): +def dream_server_loop(gen, host, port, outdir): print('\n* --web was specified, starting web server...') # Change working directory to the stable-diffusion directory os.chdir( @@ -349,7 +335,7 @@ def dream_server_loop(t2i, host, port, outdir): ) # Start server - DreamServer.model = t2i + DreamServer.model = gen # misnomer in DreamServer - this is not the model you are looking for DreamServer.outdir = outdir dream_server = ThreadingDreamServer((host, port)) print(">> Started Stable Diffusion dream server!") @@ -519,13 +505,6 @@ def create_argv_parser(): default='model', help='Indicates the Stable Diffusion model to use.', ) - parser.add_argument( - '--device', - '-d', - type=str, - default='cuda', - help="device to run stable diffusion on. defaults to cuda `torch.cuda.current_device()` if available" - ) parser.add_argument( '--model', default='stable-diffusion-1.4', diff --git a/scripts/inpaint.py b/scripts/orig_scripts/inpaint.py similarity index 100% rename from scripts/inpaint.py rename to scripts/orig_scripts/inpaint.py From 5818528aa6864eb2498ef86aba8f4f045dd606e7 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 14 Sep 2022 07:09:01 -0400 Subject: [PATCH 045/238] fix web server handling of rel and abs outdir paths (#550) * fix web server handling of rel and abs outdir paths * Can now specify either a relative or absolute path for outdir * Outdir path does not need to be inside the stable-diffusion directory * Closes security hole that allowed user to read any file within stable-diffusion (eek!) * Closes #536 --- docs/features/VARIATIONS.md | 10 +++---- docs/installation/INSTALL_MAC.md | 49 +++++++++++++++++++++----------- ldm/dream/server.py | 14 +++++---- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/docs/features/VARIATIONS.md b/docs/features/VARIATIONS.md index c1798ae759..a6c5c936c1 100644 --- a/docs/features/VARIATIONS.md +++ b/docs/features/VARIATIONS.md @@ -36,7 +36,7 @@ Outputs: The one with seed 3357757885 looks nice: - + --- @@ -65,8 +65,8 @@ variation amount used to generate it. This gives us a series of closely-related variations, including the two shown here. - - + + I like the expression on Xena's face in the first one (subseed 3647897225), and the armor on her shoulder in the second one (subseed 1614299449). Can we combine them to get the best of both worlds? @@ -81,7 +81,7 @@ Outputs: Here we are providing equal weights (0.1 and 0.1) for both the subseeds. The resulting image is close, but not exactly what I wanted: - + We could either try combining the images with different weights, or we can generate more variations around the almost-but-not-quite image. We do the latter, using both the `-V` (combining) and `-v` (variation strength) options. Note that we use `-n6` to generate 6 variations: @@ -98,7 +98,7 @@ Outputs: This produces six images, all slight variations on the combination of the chosen two images. Here's the one I like best: - + As you can see, this is a very powerful tool, which when combined with subprompt weighting, gives you great control over the content and quality of your generated images. diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 6f3a137efb..c000e818bb 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -152,21 +152,27 @@ You might also need to install Rust (I mention this again below). ### How many snakes are living in your computer? -Here's the reason why you have to specify which python to use. -There are several versions of python on macOS and the computer is -picking the wrong one. More specifically, preload_models.py and dream.py says to -find the first `python3` in the path environment variable. You can see which one -it is picking with `which python3`. These are the mostly likely paths you'll see. +You might have multiple Python installations on your system, in which case it's +important to be explicit and consistent about which one to use for a given project. +This is because virtual environments are coupled to the Python that created it (and all +the associated 'system-level' modules). + +When you run `python` or `python3`, your shell searches the colon-delimited locations +in the `PATH` environment variable (`echo $PATH` to see that list) in that order - first match wins. +You can ask for the location of the first `python3` found in your `PATH` with the `which` command like this: % which python3 /usr/bin/python3 -The above path is part of the OS. However, that path is a stub that asks you if -you want to install Xcode. If you have Xcode installed already, -/usr/bin/python3 will execute /Library/Developer/CommandLineTools/usr/bin/python3 or -/Applications/Xcode.app/Contents/Developer/usr/bin/python3 (depending on which +Anything in `/usr/bin` is [part of the OS](https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW6). However, `/usr/bin/python3` is not actually python3, but +rather a stub that offers to install Xcode (which includes python 3). If you have Xcode installed already, +`/usr/bin/python3` will execute `/Library/Developer/CommandLineTools/usr/bin/python3` or +`/Applications/Xcode.app/Contents/Developer/usr/bin/python3` (depending on which Xcode you've selected with `xcode-select`). +Note that `/usr/bin/python` is an entirely different python - specifically, python 2. Note: starting in +macOS 12.3, `/usr/bin/python` no longer exists. + % which python3 /opt/homebrew/bin/python3 @@ -176,17 +182,21 @@ for Homebrew binaries before system ones, you'll see the above path. % which python /opt/anaconda3/bin/python -If you drop the "3" you get an entirely different python. Note: starting in -macOS 12.3, /usr/bin/python no longer exists (it was python 2 anyway). - -If you have Anaconda installed, this is what you'll see. There is a -/opt/anaconda3/bin/python3 also. +If you have Anaconda installed, you will see the above path. There is a +`/opt/anaconda3/bin/python3` also. We expect that `/opt/anaconda3/bin/python` +and `/opt/anaconda3/bin/python3` should actually be the *same python*, which you can +verify by comparing the output of `python3 -V` and `python -V`. (ldm) % which python /Users/name/miniforge3/envs/ldm/bin/python -This is what you'll see if you have miniforge and you've correctly activated -the ldm environment. This is the goal. +The above is what you'll see if you have miniforge and you've correctly activated +the ldm environment, and you used option 2 in the setup instructions above ("no pyenv"). + + (anaconda3-2022.05) % which python + /Users/name/.pyenv/shims/python + +... and the above is what you'll see if you used option 1 ("Alongside pyenv"). It's all a mess and you should know [how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) if you want to fix it. Here's a brief hint of all the ways you can modify it @@ -201,6 +211,13 @@ if you want to fix it. Here's a brief hint of all the ways you can modify it Which one you use will depend on what you have installed except putting a file in /etc/paths.d is what I prefer to do. +Finally, to answer the question posed by this section's title, it may help to list +all of the `python` / `python3` things found in `$PATH` instead of just the one that +will be executed by default. To do that, add the `-a` switch to `which`: + + % which -a python3 + ... + ### Debugging? Tired of waiting for your renders to finish before you can see if it diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 19414f65d6..54118c7dd4 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -103,10 +103,14 @@ class DreamServer(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(bytes('{}', 'utf8')) else: - path = "." + self.path - cwd = os.path.realpath(os.getcwd()) - is_in_cwd = os.path.commonprefix((os.path.realpath(path), cwd)) == cwd - if not (is_in_cwd and os.path.exists(path)): + path_dir = os.path.dirname(self.path) + out_dir = os.path.realpath(self.outdir.rstrip('/')) + if self.path.startswith('/static/dream_web/'): + path = '.' + self.path + elif out_dir.endswith(path_dir): + file = os.path.basename(self.path) + path = os.path.join(self.outdir,file) + else: self.send_response(404) return mime_type = mimetypes.guess_type(path)[0] @@ -114,7 +118,7 @@ class DreamServer(BaseHTTPRequestHandler): self.send_response(200) self.send_header("Content-type", mime_type) self.end_headers() - with open("." + self.path, "rb") as content: + with open(path, "rb") as content: self.wfile.write(content.read()) else: self.send_response(404) From b02ea331dfba1597225ce690205a140287aa35a5 Mon Sep 17 00:00:00 2001 From: Robert Bolender Date: Wed, 14 Sep 2022 06:47:17 -0700 Subject: [PATCH 046/238] Clarify behavior of -v and -n parameters (#551) Fixes https://github.com/lstein/stable-diffusion/issues/544 --- docs/features/IMG2IMG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/IMG2IMG.md b/docs/features/IMG2IMG.md index ac560f6984..3930208e62 100644 --- a/docs/features/IMG2IMG.md +++ b/docs/features/IMG2IMG.md @@ -16,7 +16,7 @@ modified, ranging from `0.0` (keep the original intact), to `1.0` (ignore the original completely). The default is `0.75`, and ranges from `0.25-0.75` give interesting results. -You may also pass a `-v` option to generate count variants on +You may also pass a `-v` option to generate `-n` count variants on the original image. This is done by passing the first generated image back into img2img the requested number of times. It generates interesting variants. From dd3fff1d3e4f2b2bd6890ab09b65429b60c652b1 Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Thu, 15 Sep 2022 01:10:33 +0300 Subject: [PATCH 047/238] ~7% speedup by switch to += in ldm.modules.attention. (#569) Tested on 8GB eGPU nvidia setup so YMMV. Re-land with .clone() fix, context #508 --- ldm/modules/attention.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 55e5b9f8a4..ec96230b46 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -281,9 +281,9 @@ class BasicTransformerBlock(nn.Module): def _forward(self, x, context=None): x = x.contiguous() if x.device.type == 'mps' else x - x = self.attn1(self.norm1(x)) + x - x = self.attn2(self.norm2(x), context=context) + x - x = self.ff(self.norm3(x)) + x + x += self.attn1(self.norm1(x.clone())) + x += self.attn2(self.norm2(x.clone()), context=context) + x += self.ff(self.norm3(x.clone())) return x From df4d1162b54ff9baad5707679cf6ad7979719666 Mon Sep 17 00:00:00 2001 From: "Claus F. Strasburger" Date: Thu, 15 Sep 2022 13:21:17 +0200 Subject: [PATCH 048/238] docs: VARIATIONS.md used wrong syntax in examples (#589) --- docs/features/VARIATIONS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/VARIATIONS.md b/docs/features/VARIATIONS.md index a6c5c936c1..99b891576d 100644 --- a/docs/features/VARIATIONS.md +++ b/docs/features/VARIATIONS.md @@ -74,7 +74,7 @@ We combine the two variations using `-V` (--with_variations). Again, we must pro this to work. ``` -dream> "prompt" -S3357757885 -V3647897225,0.1;1614299449,0.1 +dream> "prompt" -S3357757885 -V3647897225:0.1,1614299449:0.1 Outputs: ./outputs/Xena/000003.1614299449.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1 -S3357757885 ``` @@ -86,7 +86,7 @@ Here we are providing equal weights (0.1 and 0.1) for both the subseeds. The res We could either try combining the images with different weights, or we can generate more variations around the almost-but-not-quite image. We do the latter, using both the `-V` (combining) and `-v` (variation strength) options. Note that we use `-n6` to generate 6 variations: ``` -dream> "prompt" -S3357757885 -V3647897225,0.1;1614299449,0.1 -v0.05 -n6 +dream> "prompt" -S3357757885 -V3647897225:0.1,1614299449:0.1 -v0.05 -n6 Outputs: ./outputs/Xena/000004.3279757577.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1,3279757577:0.05 -S3357757885 ./outputs/Xena/000004.2853129515.png: "prompt" -s50 -W512 -H512 -C7.5 -Ak_lms -V 3647897225:0.1,1614299449:0.1,2853129515:0.05 -S3357757885 From 30e69f8b32fbe90d35bcc73d3cabdac8693b033b Mon Sep 17 00:00:00 2001 From: William Becher Date: Thu, 15 Sep 2022 08:40:27 -0300 Subject: [PATCH 049/238] Fix image location on webpage - windows (#568) --- ldm/dream/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 54118c7dd4..96ffd7f2ef 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -107,7 +107,7 @@ class DreamServer(BaseHTTPRequestHandler): out_dir = os.path.realpath(self.outdir.rstrip('/')) if self.path.startswith('/static/dream_web/'): path = '.' + self.path - elif out_dir.endswith(path_dir): + elif out_dir.replace('\\', '/').endswith(path_dir): file = os.path.basename(self.path) path = os.path.join(self.outdir,file) else: From ccb2b7c2fbfda243ec7fd3c523e9f6eb5dd46c65 Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Thu, 15 Sep 2022 14:41:24 +0300 Subject: [PATCH 050/238] Use cuda only when available in main.py. (#567) Allows testing textual inversion / training flow on cpu only (very slow though). Context: #508 --- main.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index c45194db44..72aaa49c3b 100644 --- a/main.py +++ b/main.py @@ -40,7 +40,8 @@ def load_model_from_config(config, ckpt, verbose=False): print('unexpected keys:') print(u) - model.cuda() + if torch.cuda.is_available(): + model.cuda() return model @@ -549,23 +550,26 @@ class CUDACallback(Callback): # see https://github.com/SeanNaren/minGPT/blob/master/mingpt/callback.py def on_train_epoch_start(self, trainer, pl_module): # Reset the memory use counter - torch.cuda.reset_peak_memory_stats(trainer.root_gpu) - torch.cuda.synchronize(trainer.root_gpu) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats(trainer.root_gpu) + torch.cuda.synchronize(trainer.root_gpu) self.start_time = time.time() def on_train_epoch_end(self, trainer, pl_module, outputs): - torch.cuda.synchronize(trainer.root_gpu) - max_memory = ( - torch.cuda.max_memory_allocated(trainer.root_gpu) / 2**20 - ) + if torch.cuda.is_available(): + torch.cuda.synchronize(trainer.root_gpu) epoch_time = time.time() - self.start_time try: - max_memory = trainer.training_type_plugin.reduce(max_memory) epoch_time = trainer.training_type_plugin.reduce(epoch_time) - rank_zero_info(f'Average Epoch time: {epoch_time:.2f} seconds') - rank_zero_info(f'Average Peak memory {max_memory:.2f}MiB') + + if torch.cuda.is_available(): + max_memory = ( + torch.cuda.max_memory_allocated(trainer.root_gpu) / 2**20 + ) + max_memory = trainer.training_type_plugin.reduce(max_memory) + rank_zero_info(f'Average Peak memory {max_memory:.2f}MiB') except AttributeError: pass @@ -872,7 +876,6 @@ if __name__ == '__main__': config.data.params.validation.params.data_root = opt.data_root data = instantiate_from_config(config.data) - data = instantiate_from_config(config.data) # NOTE according to https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html # calling these ourselves should not be necessary but it is. # lightning still takes care of proper multiprocessing though From 9df743e2bf54829eabe87fbbb1f4aaa1c75ba0c1 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 15 Sep 2022 07:43:43 -0400 Subject: [PATCH 051/238] Web cleanup (#539) * Refactor generate.py and dream.py * config file path (models.yaml) is parsed inside Generate() to simplify API * Better handling of keyboard interrupts in file loading mode vs interactive * Removed oodles of unused variables. * move nonfunctional inpainting out of the scripts directory * fix ugly ddim tqdm formatting * fix embiggen breakage, formatting fixes --- ldm/dream/server.py | 7 +++++++ scripts/dream.py | 2 ++ static/dream_web/index.css | 14 +++++++++++--- static/dream_web/index.html | 27 +++++++++++++++------------ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 96ffd7f2ef..b56e21f6c7 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -22,6 +22,12 @@ def build_opt(post_data, seed, gfpgan_model_exists): setattr(opt, 'invert_mask', 'invert_mask' in post_data) setattr(opt, 'cfg_scale', float(post_data['cfg_scale'])) setattr(opt, 'sampler_name', post_data['sampler_name']) + + # embiggen not practical at this point because we have no way of feeding images back into img2img + # however, this code is here against that eventuality + setattr(opt, 'embiggen', None) + setattr(opt, 'embiggen_tiles', None) + setattr(opt, 'gfpgan_strength', float(post_data['gfpgan_strength']) if gfpgan_model_exists else 0) setattr(opt, 'upscale', [int(post_data['upscale_level']), float(post_data['upscale_strength'])] if post_data['upscale_level'] != '' else None) setattr(opt, 'progress_images', 'progress_images' in post_data) @@ -155,6 +161,7 @@ class DreamServer(BaseHTTPRequestHandler): def image_done(image, seed, upscaled=False): name = f'{prefix}.{seed}.png' iter_opt = argparse.Namespace(**vars(opt)) # copy + print(f'iter_opt = {iter_opt}') if opt.variation_amount > 0: this_variation = [[seed, opt.variation_amount]] if opt.with_variations is None: diff --git a/scripts/dream.py b/scripts/dream.py index aec27506c3..4044af7cb8 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -620,6 +620,7 @@ def create_cmd_parser(): ) parser.add_argument( '-embiggen', + '--embiggen', nargs='+', default=None, type=float, @@ -627,6 +628,7 @@ def create_cmd_parser(): ) parser.add_argument( '-embiggen_tiles', + '--embiggen_tiles', nargs='+', default=None, type=int, diff --git a/static/dream_web/index.css b/static/dream_web/index.css index cbe753d423..51f0f267c3 100644 --- a/static/dream_web/index.css +++ b/static/dream_web/index.css @@ -91,6 +91,7 @@ header h1 { } #fieldset-config { line-height:2em; + background-color: #F0F0F0; } input[type="number"] { width: 60px; @@ -122,6 +123,9 @@ label { cursor: pointer; color: red; } +#basic-parameters { + background-color: #EEEEEE; +} #txt2img { background-color: #DCDCDC; } @@ -129,15 +133,19 @@ label { background-color: #EEEEEE; } #img2img { - background-color: #F5F5F5; + background-color: #DCDCDC; } #gfpgan { - background-color: #DCDCDC; + background-color: #EEEEEE; } #progress-section { background-color: #F5F5F5; } - +.section-header { + text-align: left; + font-weight: bold; + padding: 0 0 0 0; +} #no-results-message:not(:only-child) { display: none; } diff --git a/static/dream_web/index.html b/static/dream_web/index.html index 628cfb4b6c..1e194c0205 100644 --- a/static/dream_web/index.html +++ b/static/dream_web/index.html @@ -25,6 +25,7 @@
+
Basic options
@@ -39,11 +40,11 @@ - + - +
-
+ -
-
- - - + +
+
+
Image-to-image options
+ + +
- - +
+
Post-processing options
- + + {validValues.map((opt) => { + return typeof opt === 'string' || + typeof opt === 'number' ? ( + + ) : ( + + ); + })} + + + + ); +}; + +export default SDSelect; diff --git a/frontend/src/components/SDSwitch.tsx b/frontend/src/components/SDSwitch.tsx new file mode 100644 index 0000000000..df2b811a2b --- /dev/null +++ b/frontend/src/components/SDSwitch.tsx @@ -0,0 +1,42 @@ +import { + Flex, + FormControl, + FormLabel, + Switch, + SwitchProps, +} from '@chakra-ui/react'; + +interface Props extends SwitchProps { + label?: string; + width?: string | number; +} + +const SDSwitch = (props: Props) => { + const { + label, + isDisabled = false, + fontSize = 'md', + size = 'md', + width, + ...rest + } = props; + return ( + + + {label && ( + + {label} + + )} + + + + ); +}; + +export default SDSwitch; diff --git a/frontend/src/features/gallery/CurrentImage.tsx b/frontend/src/features/gallery/CurrentImage.tsx new file mode 100644 index 0000000000..e2698dd09d --- /dev/null +++ b/frontend/src/features/gallery/CurrentImage.tsx @@ -0,0 +1,161 @@ +import { Center, Flex, Image, useColorModeValue } from '@chakra-ui/react'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import { setAllParameters, setInitialImagePath, setSeed } from '../sd/sdSlice'; +import { useState } from 'react'; +import ImageMetadataViewer from './ImageMetadataViewer'; +import DeleteImageModalButton from './DeleteImageModalButton'; +import SDButton from '../../components/SDButton'; +import { runESRGAN, runGFPGAN } from '../../app/socketio'; +import { createSelector } from '@reduxjs/toolkit'; +import { SystemState } from '../system/systemSlice'; +import { isEqual } from 'lodash'; + +const height = 'calc(100vh - 238px)'; + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { + isProcessing: system.isProcessing, + isConnected: system.isConnected, + isGFPGANAvailable: system.isGFPGANAvailable, + isESRGANAvailable: system.isESRGANAvailable, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const CurrentImage = () => { + const { currentImage, intermediateImage } = useAppSelector( + (state: RootState) => state.gallery + ); + const { isProcessing, isConnected, isGFPGANAvailable, isESRGANAvailable } = + useAppSelector(systemSelector); + + const dispatch = useAppDispatch(); + + const bgColor = useColorModeValue( + 'rgba(255, 255, 255, 0.85)', + 'rgba(0, 0, 0, 0.8)' + ); + + const [shouldShowImageDetails, setShouldShowImageDetails] = + useState(false); + + const imageToDisplay = intermediateImage || currentImage; + + return ( + + {imageToDisplay && ( + + + dispatch(setInitialImagePath(imageToDisplay.url)) + } + /> + + + dispatch(setAllParameters(imageToDisplay.metadata)) + } + /> + + + dispatch(setSeed(imageToDisplay.metadata.seed!)) + } + /> + + dispatch(runESRGAN(imageToDisplay))} + /> + dispatch(runGFPGAN(imageToDisplay))} + /> + + setShouldShowImageDetails(!shouldShowImageDetails) + } + /> + + + + + )} +
+ {imageToDisplay && ( + + )} + {imageToDisplay && shouldShowImageDetails && ( + + + + )} +
+
+ ); +}; + +export default CurrentImage; diff --git a/frontend/src/features/gallery/DeleteImageModalButton.tsx b/frontend/src/features/gallery/DeleteImageModalButton.tsx new file mode 100644 index 0000000000..eca53c8e9f --- /dev/null +++ b/frontend/src/features/gallery/DeleteImageModalButton.tsx @@ -0,0 +1,94 @@ +import { + IconButtonProps, + Modal, + ModalBody, + ModalCloseButton, + ModalContent, + ModalFooter, + ModalHeader, + ModalOverlay, + Text, + useDisclosure, +} from '@chakra-ui/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { + cloneElement, + ReactElement, + SyntheticEvent, +} from 'react'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { deleteImage } from '../../app/socketio'; +import { RootState } from '../../app/store'; +import SDButton from '../../components/SDButton'; +import { setShouldConfirmOnDelete, SystemState } from '../system/systemSlice'; +import { SDImage } from './gallerySlice'; + +interface Props extends IconButtonProps { + image: SDImage; + 'aria-label': string; + children: ReactElement; +} + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => system.shouldConfirmOnDelete +); + +/* +TODO: The modal and button to open it should be two different components, +but their state is closely related and I'm not sure how best to accomplish it. +*/ +const DeleteImageModalButton = (props: Omit) => { + const { isOpen, onOpen, onClose } = useDisclosure(); + const dispatch = useAppDispatch(); + const shouldConfirmOnDelete = useAppSelector(systemSelector); + + const handleClickDelete = (e: SyntheticEvent) => { + e.stopPropagation(); + shouldConfirmOnDelete ? onOpen() : handleDelete(); + }; + + const { image, children } = props; + + const handleDelete = () => { + dispatch(deleteImage(image)); + onClose(); + }; + + const handleDeleteAndDontAsk = () => { + dispatch(deleteImage(image)); + dispatch(setShouldConfirmOnDelete(false)); + onClose(); + }; + + return ( + <> + {cloneElement(children, { + onClick: handleClickDelete, + })} + + + + + Are you sure you want to delete this image? + + + It will be deleted forever! + + + + + + + + + + + ); +}; + +export default DeleteImageModalButton; diff --git a/frontend/src/features/gallery/ImageMetadataViewer.tsx b/frontend/src/features/gallery/ImageMetadataViewer.tsx new file mode 100644 index 0000000000..3f051f59b6 --- /dev/null +++ b/frontend/src/features/gallery/ImageMetadataViewer.tsx @@ -0,0 +1,124 @@ +import { + Center, + Flex, + IconButton, + Link, + List, + ListItem, + Text, +} from '@chakra-ui/react'; +import { FaPlus } from 'react-icons/fa'; +import { PARAMETERS } from '../../app/constants'; +import { useAppDispatch } from '../../app/hooks'; +import SDButton from '../../components/SDButton'; +import { setAllParameters, setParameter } from '../sd/sdSlice'; +import { SDImage, SDMetadata } from './gallerySlice'; + +type Props = { + image: SDImage; +}; + +const ImageMetadataViewer = ({ image }: Props) => { + const dispatch = useAppDispatch(); + + const keys = Object.keys(PARAMETERS); + + const metadata: Array<{ + label: string; + key: string; + value: string | number | boolean; + }> = []; + + keys.forEach((key) => { + const value = image.metadata[key as keyof SDMetadata]; + if (value !== undefined) { + metadata.push({ label: PARAMETERS[key], key, value }); + } + }); + + return ( + + dispatch(setAllParameters(image.metadata))} + /> + + File: + + {image.url} + + + {metadata.length ? ( + <> + + {metadata.map((parameter, i) => { + const { label, key, value } = parameter; + return ( + + + } + size={'xs'} + onClick={() => + dispatch( + setParameter({ + key, + value, + }) + ) + } + /> + + {label}: + + + {value === undefined || + value === null || + value === '' || + value === 0 ? ( + + None + + ) : ( + + {value.toString()} + + )} + + + ); + })} + + + Raw: + + {JSON.stringify(image.metadata)} + + + + ) : ( +
+ + No metadata available + +
+ )} +
+ ); +}; + +export default ImageMetadataViewer; diff --git a/frontend/src/features/gallery/ImageRoll.tsx b/frontend/src/features/gallery/ImageRoll.tsx new file mode 100644 index 0000000000..b624db3aaa --- /dev/null +++ b/frontend/src/features/gallery/ImageRoll.tsx @@ -0,0 +1,150 @@ +import { + Box, + Flex, + Icon, + IconButton, + Image, + useColorModeValue, +} from '@chakra-ui/react'; +import { RootState } from '../../app/store'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { SDImage, setCurrentImage } from './gallerySlice'; +import { FaCheck, FaCopy, FaSeedling, FaTrash } from 'react-icons/fa'; +import DeleteImageModalButton from './DeleteImageModalButton'; +import { memo, SyntheticEvent, useState } from 'react'; +import { setAllParameters, setSeed } from '../sd/sdSlice'; + +interface HoverableImageProps { + image: SDImage; + isSelected: boolean; +} + +const HoverableImage = memo( + (props: HoverableImageProps) => { + const [isHovered, setIsHovered] = useState(false); + const dispatch = useAppDispatch(); + + const checkColor = useColorModeValue('green.600', 'green.300'); + const bgColor = useColorModeValue('gray.200', 'gray.700'); + const bgGradient = useColorModeValue( + 'radial-gradient(circle, rgba(255,255,255,0.7) 0%, rgba(255,255,255,0.7) 20%, rgba(0,0,0,0) 100%)', + 'radial-gradient(circle, rgba(0,0,0,0.7) 0%, rgba(0,0,0,0.7) 20%, rgba(0,0,0,0) 100%)' + ); + + const { image, isSelected } = props; + const { url, uuid, metadata } = image; + + const handleMouseOver = () => setIsHovered(true); + const handleMouseOut = () => setIsHovered(false); + const handleClickSetAllParameters = (e: SyntheticEvent) => { + e.stopPropagation(); + dispatch(setAllParameters(metadata)); + }; + const handleClickSetSeed = (e: SyntheticEvent) => { + e.stopPropagation(); + dispatch(setSeed(image.metadata.seed!)); // component not rendered unless this exists + }; + + return ( + + + dispatch(setCurrentImage(image))} + onMouseOver={handleMouseOver} + onMouseOut={handleMouseOut} + > + {isSelected && ( + + )} + {isHovered && ( + + + } + size='xs' + fontSize={15} + /> + + } + size='xs' + fontSize={15} + onClickCapture={handleClickSetAllParameters} + /> + {image.metadata.seed && ( + } + size='xs' + fontSize={16} + onClickCapture={handleClickSetSeed} + /> + )} + + )} + + + ); + }, + (prev, next) => + prev.image.uuid === next.image.uuid && + prev.isSelected === next.isSelected +); + +const ImageRoll = () => { + const { images, currentImageUuid } = useAppSelector( + (state: RootState) => state.gallery + ); + + return ( + + {[...images].reverse().map((image) => { + const { uuid } = image; + const isSelected = currentImageUuid === uuid; + return ( + + ); + })} + + ); +}; + +export default ImageRoll; diff --git a/frontend/src/features/gallery/gallerySlice.ts b/frontend/src/features/gallery/gallerySlice.ts new file mode 100644 index 0000000000..3c7611724f --- /dev/null +++ b/frontend/src/features/gallery/gallerySlice.ts @@ -0,0 +1,144 @@ +import { createSlice } from '@reduxjs/toolkit'; +import type { PayloadAction } from '@reduxjs/toolkit'; +import { v4 as uuidv4 } from 'uuid'; +import { UpscalingLevel } from '../sd/sdSlice'; +import { backendToFrontendParameters } from '../../app/parameterTranslation'; + +// TODO: Revise pending metadata RFC: https://github.com/lstein/stable-diffusion/issues/266 +export interface SDMetadata { + prompt?: string; + steps?: number; + cfgScale?: number; + height?: number; + width?: number; + sampler?: string; + seed?: number; + img2imgStrength?: number; + gfpganStrength?: number; + upscalingLevel?: UpscalingLevel; + upscalingStrength?: number; + initialImagePath?: string; + maskPath?: string; + seamless?: boolean; + shouldFitToWidthHeight?: boolean; +} + +export interface SDImage { + // TODO: I have installed @types/uuid but cannot figure out how to use them here. + uuid: string; + url: string; + metadata: SDMetadata; +} + +export interface GalleryState { + currentImageUuid: string; + images: Array; + intermediateImage?: SDImage; + currentImage?: SDImage; +} + +const initialState: GalleryState = { + currentImageUuid: '', + images: [], +}; + +export const gallerySlice = createSlice({ + name: 'gallery', + initialState, + reducers: { + setCurrentImage: (state, action: PayloadAction) => { + state.currentImage = action.payload; + state.currentImageUuid = action.payload.uuid; + }, + removeImage: (state, action: PayloadAction) => { + const { uuid } = action.payload; + + const newImages = state.images.filter((image) => image.uuid !== uuid); + + const imageToDeleteIndex = state.images.findIndex( + (image) => image.uuid === uuid + ); + + const newCurrentImageIndex = Math.min( + Math.max(imageToDeleteIndex, 0), + newImages.length - 1 + ); + + state.images = newImages; + + state.currentImage = newImages.length + ? newImages[newCurrentImageIndex] + : undefined; + + state.currentImageUuid = newImages.length + ? newImages[newCurrentImageIndex].uuid + : ''; + }, + addImage: (state, action: PayloadAction) => { + state.images.push(action.payload); + state.currentImageUuid = action.payload.uuid; + state.intermediateImage = undefined; + state.currentImage = action.payload; + }, + setIntermediateImage: (state, action: PayloadAction) => { + state.intermediateImage = action.payload; + }, + clearIntermediateImage: (state) => { + state.intermediateImage = undefined; + }, + setGalleryImages: ( + state, + action: PayloadAction< + Array<{ + path: string; + metadata: { [key: string]: string | number | boolean }; + }> + > + ) => { + // TODO: Revise pending metadata RFC: https://github.com/lstein/stable-diffusion/issues/266 + const images = action.payload; + + if (images.length === 0) { + // there are no images on disk, clear the gallery + state.images = []; + state.currentImageUuid = ''; + state.currentImage = undefined; + } else { + // Filter image urls that are already in the rehydrated state + const filteredImages = action.payload.filter( + (image) => !state.images.find((i) => i.url === image.path) + ); + + const preparedImages = filteredImages.map((image): SDImage => { + return { + uuid: uuidv4(), + url: image.path, + metadata: backendToFrontendParameters(image.metadata), + }; + }); + + const newImages = [...state.images].concat(preparedImages); + + // if previous currentimage no longer exists, set a new one + if (!newImages.find((image) => image.uuid === state.currentImageUuid)) { + const newCurrentImage = newImages[newImages.length - 1]; + state.currentImage = newCurrentImage; + state.currentImageUuid = newCurrentImage.uuid; + } + + state.images = newImages; + } + }, + }, +}); + +export const { + setCurrentImage, + removeImage, + addImage, + setGalleryImages, + setIntermediateImage, + clearIntermediateImage, +} = gallerySlice.actions; + +export default gallerySlice.reducer; diff --git a/frontend/src/features/header/ProgressBar.tsx b/frontend/src/features/header/ProgressBar.tsx new file mode 100644 index 0000000000..28d7b44c63 --- /dev/null +++ b/frontend/src/features/header/ProgressBar.tsx @@ -0,0 +1,35 @@ +import { Progress } from '@chakra-ui/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; +import { useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import { SDState } from '../sd/sdSlice'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + realSteps: sd.realSteps, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const ProgressBar = () => { + const { realSteps } = useAppSelector(sdSelector); + const { currentStep } = useAppSelector((state: RootState) => state.system); + const progress = Math.round((currentStep * 100) / realSteps); + return ( + + ); +}; + +export default ProgressBar; diff --git a/frontend/src/features/header/SiteHeader.tsx b/frontend/src/features/header/SiteHeader.tsx new file mode 100644 index 0000000000..f950eea2c0 --- /dev/null +++ b/frontend/src/features/header/SiteHeader.tsx @@ -0,0 +1,93 @@ +import { + Flex, + Heading, + IconButton, + Link, + Spacer, + Text, + useColorMode, +} from '@chakra-ui/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; + +import { FaSun, FaMoon, FaGithub } from 'react-icons/fa'; +import { MdHelp, MdSettings } from 'react-icons/md'; +import { useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import SettingsModal from '../system/SettingsModal'; +import { SystemState } from '../system/systemSlice'; + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { isConnected: system.isConnected }; + }, + { + memoizeOptions: { resultEqualityCheck: isEqual }, + } +); + +const SiteHeader = () => { + const { colorMode, toggleColorMode } = useColorMode(); + const { isConnected } = useAppSelector(systemSelector); + + return ( + + Stable Diffusion Dream Server + + + + + {isConnected ? `Connected to server` : 'No connection to server'} + + + + } + /> + + + + + + } + /> + + + + + } + /> + + : } + /> + + ); +}; + +export default SiteHeader; diff --git a/frontend/src/features/sd/ESRGANOptions.tsx b/frontend/src/features/sd/ESRGANOptions.tsx new file mode 100644 index 0000000000..928215523b --- /dev/null +++ b/frontend/src/features/sd/ESRGANOptions.tsx @@ -0,0 +1,84 @@ +import { Flex } from '@chakra-ui/react'; + +import { RootState } from '../../app/store'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; + +import { + setUpscalingLevel, + setUpscalingStrength, + UpscalingLevel, + SDState, +} from '../sd/sdSlice'; + +import SDNumberInput from '../../components/SDNumberInput'; +import SDSelect from '../../components/SDSelect'; + +import { UPSCALING_LEVELS } from '../../app/constants'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; +import { SystemState } from '../system/systemSlice'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + upscalingLevel: sd.upscalingLevel, + upscalingStrength: sd.upscalingStrength, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { + isESRGANAvailable: system.isESRGANAvailable, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); +const ESRGANOptions = () => { + const { upscalingLevel, upscalingStrength } = useAppSelector(sdSelector); + + const { isESRGANAvailable } = useAppSelector(systemSelector); + + const dispatch = useAppDispatch(); + + return ( + + + dispatch( + setUpscalingLevel( + Number(e.target.value) as UpscalingLevel + ) + ) + } + validValues={UPSCALING_LEVELS} + /> + dispatch(setUpscalingStrength(Number(v)))} + value={upscalingStrength} + /> + + ); +}; + +export default ESRGANOptions; diff --git a/frontend/src/features/sd/GFPGANOptions.tsx b/frontend/src/features/sd/GFPGANOptions.tsx new file mode 100644 index 0000000000..ae6a00de40 --- /dev/null +++ b/frontend/src/features/sd/GFPGANOptions.tsx @@ -0,0 +1,63 @@ +import { Flex } from '@chakra-ui/react'; + +import { RootState } from '../../app/store'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; + +import { SDState, setGfpganStrength } from '../sd/sdSlice'; + +import SDNumberInput from '../../components/SDNumberInput'; + +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; +import { SystemState } from '../system/systemSlice'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + gfpganStrength: sd.gfpganStrength, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { + isGFPGANAvailable: system.isGFPGANAvailable, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); +const GFPGANOptions = () => { + const { gfpganStrength } = useAppSelector(sdSelector); + + const { isGFPGANAvailable } = useAppSelector(systemSelector); + + const dispatch = useAppDispatch(); + + return ( + + dispatch(setGfpganStrength(Number(v)))} + value={gfpganStrength} + /> + + ); +}; + +export default GFPGANOptions; diff --git a/frontend/src/features/sd/ImageToImageOptions.tsx b/frontend/src/features/sd/ImageToImageOptions.tsx new file mode 100644 index 0000000000..379a8d231c --- /dev/null +++ b/frontend/src/features/sd/ImageToImageOptions.tsx @@ -0,0 +1,54 @@ +import { Flex } from '@chakra-ui/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import SDNumberInput from '../../components/SDNumberInput'; +import SDSwitch from '../../components/SDSwitch'; +import InitImage from './InitImage'; +import { + SDState, + setImg2imgStrength, + setShouldFitToWidthHeight, +} from './sdSlice'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + initialImagePath: sd.initialImagePath, + img2imgStrength: sd.img2imgStrength, + shouldFitToWidthHeight: sd.shouldFitToWidthHeight, + }; + } +); + +const ImageToImageOptions = () => { + const { initialImagePath, img2imgStrength, shouldFitToWidthHeight } = + useAppSelector(sdSelector); + + const dispatch = useAppDispatch(); + return ( + + dispatch(setImg2imgStrength(Number(v)))} + value={img2imgStrength} + /> + + dispatch(setShouldFitToWidthHeight(e.target.checked)) + } + /> + + + ); +}; + +export default ImageToImageOptions; diff --git a/frontend/src/features/sd/InitImage.css b/frontend/src/features/sd/InitImage.css new file mode 100644 index 0000000000..31fe87fa17 --- /dev/null +++ b/frontend/src/features/sd/InitImage.css @@ -0,0 +1,20 @@ +.checkerboard { + background-position: 0px 0px, 10px 10px; + background-size: 20px 20px; + background-image: linear-gradient( + 45deg, + #eee 25%, + transparent 25%, + transparent 75%, + #eee 75%, + #eee 100% + ), + linear-gradient( + 45deg, + #eee 25%, + white 25%, + white 75%, + #eee 75%, + #eee 100% + ); +} diff --git a/frontend/src/features/sd/InitImage.tsx b/frontend/src/features/sd/InitImage.tsx new file mode 100644 index 0000000000..5d7c0aa2d4 --- /dev/null +++ b/frontend/src/features/sd/InitImage.tsx @@ -0,0 +1,155 @@ +import { + Button, + Flex, + IconButton, + Image, + useToast, +} from '@chakra-ui/react'; +import { SyntheticEvent, useCallback, useState } from 'react'; +import { FileRejection, useDropzone } from 'react-dropzone'; +import { FaTrash } from 'react-icons/fa'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import { + SDState, + setInitialImagePath, + setMaskPath, +} from '../../features/sd/sdSlice'; +import MaskUploader from './MaskUploader'; +import './InitImage.css'; +import { uploadInitialImage } from '../../app/socketio'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + initialImagePath: sd.initialImagePath, + maskPath: sd.maskPath, + }; + }, + { memoizeOptions: { resultEqualityCheck: isEqual } } +); + +const InitImage = () => { + const toast = useToast(); + const dispatch = useAppDispatch(); + const { initialImagePath, maskPath } = useAppSelector(sdSelector); + + const onDrop = useCallback( + (acceptedFiles: Array, fileRejections: Array) => { + fileRejections.forEach((rejection: FileRejection) => { + const msg = rejection.errors.reduce( + (acc: string, cur: { message: string }) => acc + '\n' + cur.message, + '' + ); + + toast({ + title: 'Upload failed', + description: msg, + status: 'error', + isClosable: true, + }); + }); + + acceptedFiles.forEach((file: File) => { + dispatch(uploadInitialImage(file)); + }); + }, + [dispatch, toast] + ); + + const { getRootProps, getInputProps, open } = useDropzone({ + onDrop, + accept: { + 'image/jpeg': ['.jpg', '.jpeg', '.png'], + }, + }); + + const [shouldShowMask, setShouldShowMask] = useState(false); + const handleClickUploadIcon = (e: SyntheticEvent) => { + e.stopPropagation(); + open(); + }; + const handleClickResetInitialImageAndMask = (e: SyntheticEvent) => { + e.stopPropagation(); + dispatch(setInitialImagePath('')); + dispatch(setMaskPath('')); + }; + + const handleMouseOverInitialImageUploadButton = () => + setShouldShowMask(false); + const handleMouseOutInitialImageUploadButton = () => setShouldShowMask(true); + + const handleMouseOverMaskUploadButton = () => setShouldShowMask(true); + const handleMouseOutMaskUploadButton = () => setShouldShowMask(true); + + return ( + e.stopPropagation() : undefined, + })} + direction={'column'} + alignItems={'center'} + gap={2} + > + + + + + + + + } + /> + + {initialImagePath && ( + + + {shouldShowMask && maskPath && ( + + )} + + )} + + ); +}; + +export default InitImage; diff --git a/frontend/src/features/sd/MaskUploader.tsx b/frontend/src/features/sd/MaskUploader.tsx new file mode 100644 index 0000000000..173cf1880c --- /dev/null +++ b/frontend/src/features/sd/MaskUploader.tsx @@ -0,0 +1,61 @@ +import { useToast } from '@chakra-ui/react'; +import { cloneElement, ReactElement, SyntheticEvent, useCallback } from 'react'; +import { FileRejection, useDropzone } from 'react-dropzone'; +import { useAppDispatch } from '../../app/hooks'; +import { uploadMaskImage } from '../../app/socketio'; + +type Props = { + children: ReactElement; +}; + +const MaskUploader = ({ children }: Props) => { + const dispatch = useAppDispatch(); + const toast = useToast(); + + const onDrop = useCallback( + (acceptedFiles: Array, fileRejections: Array) => { + fileRejections.forEach((rejection: FileRejection) => { + const msg = rejection.errors.reduce( + (acc: string, cur: { message: string }) => + acc + '\n' + cur.message, + '' + ); + + toast({ + title: 'Upload failed', + description: msg, + status: 'error', + isClosable: true, + }); + }); + + acceptedFiles.forEach((file: File) => { + dispatch(uploadMaskImage(file)); + }); + }, + [dispatch, toast] + ); + + const { getRootProps, getInputProps, open } = useDropzone({ + onDrop, + accept: { + 'image/jpeg': ['.jpg', '.jpeg', '.png'], + }, + }); + + const handleClickUploadIcon = (e: SyntheticEvent) => { + e.stopPropagation(); + open(); + }; + + return ( +
+ + {cloneElement(children, { + onClick: handleClickUploadIcon, + })} +
+ ); +}; + +export default MaskUploader; diff --git a/frontend/src/features/sd/OptionsAccordion.tsx b/frontend/src/features/sd/OptionsAccordion.tsx new file mode 100644 index 0000000000..d2a02450f2 --- /dev/null +++ b/frontend/src/features/sd/OptionsAccordion.tsx @@ -0,0 +1,211 @@ +import { + Flex, + Box, + Text, + Accordion, + AccordionItem, + AccordionButton, + AccordionIcon, + AccordionPanel, + Switch, +} from '@chakra-ui/react'; + +import { RootState } from '../../app/store'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; + +import { + setShouldRunGFPGAN, + setShouldRunESRGAN, + SDState, + setShouldUseInitImage, +} from '../sd/sdSlice'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; +import { setOpenAccordions, SystemState } from '../system/systemSlice'; +import SeedVariationOptions from './SeedVariationOptions'; +import SamplerOptions from './SamplerOptions'; +import ESRGANOptions from './ESRGANOptions'; +import GFPGANOptions from './GFPGANOptions'; +import OutputOptions from './OutputOptions'; +import ImageToImageOptions from './ImageToImageOptions'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + initialImagePath: sd.initialImagePath, + shouldUseInitImage: sd.shouldUseInitImage, + shouldRunESRGAN: sd.shouldRunESRGAN, + shouldRunGFPGAN: sd.shouldRunGFPGAN, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { + isGFPGANAvailable: system.isGFPGANAvailable, + isESRGANAvailable: system.isESRGANAvailable, + openAccordions: system.openAccordions, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const OptionsAccordion = () => { + const { + shouldRunESRGAN, + shouldRunGFPGAN, + shouldUseInitImage, + initialImagePath, + } = useAppSelector(sdSelector); + + const { isGFPGANAvailable, isESRGANAvailable, openAccordions } = + useAppSelector(systemSelector); + + const dispatch = useAppDispatch(); + + return ( + + dispatch(setOpenAccordions(openAccordions)) + } + > + +

+ + + Seed & Variation + + + +

+ + + +
+ +

+ + + Sampler + + + +

+ + + +
+ +

+ + + Upscale (ESRGAN) + + dispatch( + setShouldRunESRGAN(e.target.checked) + ) + } + /> + + + +

+ + + +
+ +

+ + + Fix Faces (GFPGAN) + + dispatch( + setShouldRunGFPGAN(e.target.checked) + ) + } + /> + + + +

+ + + +
+ +

+ + + Image to Image + + dispatch( + setShouldUseInitImage(e.target.checked) + ) + } + /> + + + +

+ + + +
+ +

+ + + Output + + + +

+ + + +
+
+ ); +}; + +export default OptionsAccordion; diff --git a/frontend/src/features/sd/OutputOptions.tsx b/frontend/src/features/sd/OutputOptions.tsx new file mode 100644 index 0000000000..abf8acd114 --- /dev/null +++ b/frontend/src/features/sd/OutputOptions.tsx @@ -0,0 +1,66 @@ +import { Flex } from '@chakra-ui/react'; + +import { RootState } from '../../app/store'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; + +import { setHeight, setWidth, setSeamless, SDState } from '../sd/sdSlice'; + +import SDSelect from '../../components/SDSelect'; + +import { HEIGHTS, WIDTHS } from '../../app/constants'; +import SDSwitch from '../../components/SDSwitch'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; + +const sdSelector = createSelector( + (state: RootState) => state.sd, + (sd: SDState) => { + return { + height: sd.height, + width: sd.width, + seamless: sd.seamless, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const OutputOptions = () => { + const { height, width, seamless } = useAppSelector(sdSelector); + + const dispatch = useAppDispatch(); + + return ( + + + dispatch(setWidth(Number(e.target.value)))} + validValues={WIDTHS} + /> + + dispatch(setHeight(Number(e.target.value))) + } + validValues={HEIGHTS} + /> + + dispatch(setSeamless(e.target.checked))} + /> + + ); +}; + +export default OutputOptions; diff --git a/frontend/src/features/sd/ProcessButtons.tsx b/frontend/src/features/sd/ProcessButtons.tsx new file mode 100644 index 0000000000..8a85d87fcb --- /dev/null +++ b/frontend/src/features/sd/ProcessButtons.tsx @@ -0,0 +1,58 @@ +import { Flex } from '@chakra-ui/react'; +import { createSelector } from '@reduxjs/toolkit'; +import { isEqual } from 'lodash'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { cancelProcessing, generateImage } from '../../app/socketio'; +import { RootState } from '../../app/store'; +import SDButton from '../../components/SDButton'; +import { SystemState } from '../system/systemSlice'; +import useCheckParameters from '../system/useCheckParameters'; + +const systemSelector = createSelector( + (state: RootState) => state.system, + (system: SystemState) => { + return { + isProcessing: system.isProcessing, + isConnected: system.isConnected, + }; + }, + { + memoizeOptions: { + resultEqualityCheck: isEqual, + }, + } +); + +const ProcessButtons = () => { + const { isProcessing, isConnected } = useAppSelector(systemSelector); + + const dispatch = useAppDispatch(); + + const isReady = useCheckParameters(); + + return ( + + dispatch(generateImage())} + /> + dispatch(cancelProcessing())} + /> + + ); +}; + +export default ProcessButtons; diff --git a/frontend/src/features/sd/PromptInput.tsx b/frontend/src/features/sd/PromptInput.tsx new file mode 100644 index 0000000000..1eeb27b293 --- /dev/null +++ b/frontend/src/features/sd/PromptInput.tsx @@ -0,0 +1,25 @@ +import { Textarea } from '@chakra-ui/react'; +import { useAppDispatch, useAppSelector } from '../../app/hooks'; +import { RootState } from '../../app/store'; +import { setPrompt } from '../sd/sdSlice'; + +const PromptInput = () => { + const { prompt } = useAppSelector((state: RootState) => state.sd); + const dispatch = useAppDispatch(); + + return ( + @@ -30,10 +32,10 @@ - - - - + + - - + + @@ -114,7 +116,7 @@
- Postprocessing...1/3 + Postprocessing...1/3
diff --git a/static/dream_web/index.js b/static/dream_web/index.js index ac68034920..3af0308fb5 100644 --- a/static/dream_web/index.js +++ b/static/dream_web/index.js @@ -1,3 +1,41 @@ +const socket = io(); + +function resetForm() { + var form = document.getElementById('generate-form'); + form.querySelector('fieldset').removeAttribute('disabled'); +} + +function initProgress(totalSteps, showProgressImages) { + // TODO: Progress could theoretically come from multiple jobs at the same time (in the future) + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'initial'; + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('max', totalSteps); + + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = BLANK_IMAGE_URL; + progressImageEle.style.display = showProgressImages ? 'initial': 'none'; +} + +function setProgress(step, totalSteps, src) { + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('value', step); + + if (src) { + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = src; + } +} + +function resetProgress(hide = true) { + if (hide) { + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'none'; + } + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('value', 0); +} + function toBase64(file) { return new Promise((resolve, reject) => { const r = new FileReader(); @@ -9,29 +47,17 @@ function toBase64(file) { function appendOutput(src, seed, config) { let outputNode = document.createElement("figure"); - - let variations = config.with_variations; - if (config.variation_amount > 0) { - variations = (variations ? variations + ',' : '') + seed + ':' + config.variation_amount; - } - let baseseed = (config.with_variations || config.variation_amount > 0) ? config.seed : seed; - let altText = baseseed + ' | ' + (variations ? variations + ' | ' : '') + config.prompt; + let altText = seed.toString() + " | " + config.prompt; - // img needs width and height for lazy loading to work const figureContents = ` - ${altText} + ${altText}
${seed}
`; outputNode.innerHTML = figureContents; - let figcaption = outputNode.querySelector('figcaption'); + let figcaption = outputNode.querySelector('figcaption') // Reload image config figcaption.addEventListener('click', () => { @@ -40,17 +66,28 @@ function appendOutput(src, seed, config) { if (k == 'initimg') { continue; } form.querySelector(`*[name=${k}]`).value = config[k]; } + if (config.variation_amount > 0 || config.with_variations != '') { + document.querySelector("#seed").value = config.seed; + } else { + document.querySelector("#seed").value = seed; + } - document.querySelector("#seed").value = baseseed; - document.querySelector("#with_variations").value = variations || ''; - if (document.querySelector("#variation_amount").value <= 0) { - document.querySelector("#variation_amount").value = 0.2; + if (config.variation_amount > 0) { + let oldVarAmt = document.querySelector("#variation_amount").value + let oldVariations = document.querySelector("#with_variations").value + let varSep = '' + document.querySelector("#variation_amount").value = 0; + if (document.querySelector("#with_variations").value != '') { + varSep = "," + } + document.querySelector("#with_variations").value = oldVariations + varSep + seed + ':' + config.variation_amount } saveFields(document.querySelector("#generate-form")); }); document.querySelector("#results").prepend(outputNode); + document.querySelector("#no-results-message")?.remove(); } function saveFields(form) { @@ -79,9 +116,8 @@ function clearFields(form) { const BLANK_IMAGE_URL = 'data:image/svg+xml,'; async function generateSubmit(form) { - const prompt = document.querySelector("#prompt").value; - // Convert file data to base64 + // TODO: Should probably uplaod files with formdata or something, and store them in the backend? let formData = Object.fromEntries(new FormData(form)); formData.initimg_name = formData.initimg.name formData.initimg = formData.initimg.name !== '' ? await toBase64(formData.initimg) : null; @@ -89,90 +125,75 @@ async function generateSubmit(form) { let strength = formData.strength; let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; - let progressSectionEle = document.querySelector('#progress-section'); - progressSectionEle.style.display = 'initial'; - let progressEle = document.querySelector('#progress-bar'); - progressEle.setAttribute('max', totalSteps); - let progressImageEle = document.querySelector('#progress-image'); - progressImageEle.src = BLANK_IMAGE_URL; + // Initialize the progress bar + initProgress(totalSteps); - progressImageEle.style.display = {}.hasOwnProperty.call(formData, 'progress_images') ? 'initial': 'none'; - - // Post as JSON, using Fetch streaming to get results + // POST, use response to listen for events fetch(form.action, { method: form.method, + headers: new Headers({'content-type': 'application/json'}), body: JSON.stringify(formData), - }).then(async (response) => { - const reader = response.body.getReader(); - - let noOutputs = true; - while (true) { - let {value, done} = await reader.read(); - value = new TextDecoder().decode(value); - if (done) { - progressSectionEle.style.display = 'none'; - break; - } - - for (let event of value.split('\n').filter(e => e !== '')) { - const data = JSON.parse(event); - - if (data.event === 'result') { - noOutputs = false; - appendOutput(data.url, data.seed, data.config); - progressEle.setAttribute('value', 0); - progressEle.setAttribute('max', totalSteps); - } else if (data.event === 'upscaling-started') { - document.getElementById("processing_cnt").textContent=data.processed_file_cnt; - document.getElementById("scaling-inprocess-message").style.display = "block"; - } else if (data.event === 'upscaling-done') { - document.getElementById("scaling-inprocess-message").style.display = "none"; - } else if (data.event === 'step') { - progressEle.setAttribute('value', data.step); - if (data.url) { - progressImageEle.src = data.url; - } - } else if (data.event === 'canceled') { - // avoid alerting as if this were an error case - noOutputs = false; - } - } - } - - // Re-enable form, remove no-results-message - form.querySelector('fieldset').removeAttribute('disabled'); - document.querySelector("#prompt").value = prompt; - document.querySelector('progress').setAttribute('value', '0'); - - if (noOutputs) { - alert("Error occurred while generating."); - } + }) + .then(response => response.json()) + .then(data => { + var dreamId = data.dreamId; + socket.emit('join_room', { 'room': dreamId }); }); - // Disable form while generating form.querySelector('fieldset').setAttribute('disabled',''); - document.querySelector("#prompt").value = `Generating: "${prompt}"`; } -async function fetchRunLog() { - try { - let response = await fetch('/run_log.json') - const data = await response.json(); - for(let item of data.run_log) { - appendOutput(item.url, item.seed, item); - } - } catch (e) { - console.error(e); - } -} +// Socket listeners +socket.on('job_started', (data) => {}) -window.onload = async () => { - document.querySelector("#prompt").addEventListener("keydown", (e) => { - if (e.key === "Enter" && !e.shiftKey) { - const form = e.target.form; - generateSubmit(form); - } - }); +socket.on('dream_result', (data) => { + var jobId = data.jobId; + var dreamId = data.dreamId; + var dreamRequest = data.dreamRequest; + var src = 'api/images/' + dreamId; + + appendOutput(src, dreamRequest.seed, dreamRequest); + + resetProgress(false); +}) + +socket.on('dream_progress', (data) => { + // TODO: it'd be nice if we could get a seed reported here, but the generator would need to be updated + var step = data.step; + var totalSteps = data.totalSteps; + var jobId = data.jobId; + var dreamId = data.dreamId; + + var progressType = data.progressType + if (progressType === 'GENERATION') { + var src = data.hasProgressImage ? + 'api/intermediates/' + dreamId + '/' + step + : null; + setProgress(step, totalSteps, src); + } else if (progressType === 'UPSCALING_STARTED') { + // step and totalSteps are used for upscale count on this message + document.getElementById("processing_cnt").textContent = step; + document.getElementById("processing_total").textContent = totalSteps; + document.getElementById("scaling-inprocess-message").style.display = "block"; + } else if (progressType == 'UPSCALING_DONE') { + document.getElementById("scaling-inprocess-message").style.display = "none"; + } +}) + +socket.on('job_canceled', (data) => { + resetForm(); + resetProgress(); +}) + +socket.on('job_done', (data) => { + jobId = data.jobId + socket.emit('leave_room', { 'room': jobId }); + + resetForm(); + resetProgress(); +}) + +window.onload = () => { document.querySelector("#generate-form").addEventListener('submit', (e) => { e.preventDefault(); const form = e.target; @@ -183,7 +204,7 @@ window.onload = async () => { saveFields(e.target.form); }); document.querySelector("#reset-seed").addEventListener('click', (e) => { - document.querySelector("#seed").value = -1; + document.querySelector("#seed").value = 0; saveFields(e.target.form); }); document.querySelector("#reset-all").addEventListener('click', (e) => { @@ -199,15 +220,8 @@ window.onload = async () => { console.error(e); }); }); - document.documentElement.addEventListener('keydown', (e) => { - if (e.key === "Escape") - fetch('/cancel').catch(err => { - console.error(err); - }); - }); if (!config.gfpgan_model_exists) { document.querySelector("#gfpgan").style.display = 'none'; } - await fetchRunLog() }; From 00d2d0e90eb45bdf87a9594eeab7be730fc3eb32 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 14:52:25 -0400 Subject: [PATCH 066/238] Flask/React web server now merged, but needs fixes. * due to changes in the metadata written to PNG files, web server cannot display images * issue is identified and will be fixed in next 24h * Python 3.9 required for flask/react web server; environment must be updated. --- backend/server.py | 8 ++++---- environment.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/server.py b/backend/server.py index db53ae45e5..8dbbc65cb1 100644 --- a/backend/server.py +++ b/backend/server.py @@ -19,8 +19,7 @@ from uuid import uuid4 from ldm.gfpgan.gfpgan_tools import real_esrgan_upscale from ldm.gfpgan.gfpgan_tools import run_gfpgan from ldm.generate import Generate -from ldm.dream.pngwriter import PngWriter, PromptFormatter - +from ldm.dream.pngwriter import PngWriter from modules.parameters import parameters_to_command, create_cmd_parser @@ -29,10 +28,11 @@ USER CONFIG """ output_dir = "outputs/" # Base output directory for images -host = 'localhost' # Web & socket.io host +#host = 'localhost' # Web & socket.io host +host = '0.0.0.0' # Web & socket.io host port = 9090 # Web & socket.io port verbose = False # enables copious socket.io logging -additional_allowed_origins = ['http://localhost:5173'] # additional CORS allowed origins +additional_allowed_origins = ['http://localhost:9090'] # additional CORS allowed origins """ diff --git a/environment.yaml b/environment.yaml index b0e2eb8090..6ca403e204 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,7 +3,7 @@ channels: - pytorch - defaults dependencies: - - python=3.8.5 + - python>=3.9 - pip=20.3 - cudatoolkit=11.3 - pytorch=1.11.0 @@ -20,7 +20,7 @@ dependencies: - realesrgan==0.2.5.0 - test-tube>=0.7.5 - streamlit==1.12.0 - - pillow==9.2.0 + - pillow==6.2.0 - einops==0.3.0 - torch-fidelity==0.3.0 - transformers==4.19.2 From cbac95b02a578df632502f66496b985bcf7ac4ae Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 16:35:34 -0400 Subject: [PATCH 067/238] Merge with PR #602 - New and improved web api - Author: @Kyle0654 --- .gitignore | 1 + docs/other/CONTRIBUTORS.md | 1 + ldm/dream/pngwriter.py | 5 +- ldm/dream/server.py | 2 +- ldm/generate.py | 2 +- requirements.txt | 5 + server/application.py | 69 ++++----- server/containers.py | 26 ++-- server/models.py | 229 ++++++++++++++++++++++------- server/services.py | 278 ++++++++++++++++++++++++++---------- server/views.py | 61 ++++++-- static/dream_web/index.css | 61 +++++--- static/dream_web/index.html | 274 ++++++++++++++++++++--------------- static/dream_web/index.js | 243 ++++++++++++++++++++++++++----- 14 files changed, 899 insertions(+), 358 deletions(-) diff --git a/.gitignore b/.gitignore index e8d7c1f189..7e4dd4bea9 100644 --- a/.gitignore +++ b/.gitignore @@ -191,3 +191,4 @@ checkpoints .scratch/ .vscode/ gfpgan/ +models/ldm/stable-diffusion-v1/model.sha256 diff --git a/docs/other/CONTRIBUTORS.md b/docs/other/CONTRIBUTORS.md index 78f33d93d1..948795d3f2 100644 --- a/docs/other/CONTRIBUTORS.md +++ b/docs/other/CONTRIBUTORS.md @@ -51,6 +51,7 @@ We thank them for all of their time and hard work. - [Any Winter](https://github.com/any-winter-4079) - [Doggettx](https://github.com/doggettx) - [Matthias Wild](https://github.com/mauwii) +- [Kyle Schouviller](https://github.com/kyle0654) ## __Original CompVis Authors:__ diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index a8d2425b91..6291059585 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -33,11 +33,12 @@ class PngWriter: # saves image named _image_ to outdir/name, writing metadata from prompt # returns full path of output - def save_image_and_prompt_to_png(self, image, dream_prompt, metadata, name): + def save_image_and_prompt_to_png(self, image, dream_prompt, name, metadata=None): path = os.path.join(self.outdir, name) info = PngImagePlugin.PngInfo() info.add_text('Dream', dream_prompt) - info.add_text('sd-metadata', json.dumps(metadata)) + if metadata: # TODO: merge command line app's method of writing metadata and always just write metadata + info.add_text('sd-metadata', json.dumps(metadata)) image.save(path, 'PNG', pnginfo=info) return path diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 003ec70533..cde3957a1f 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -230,7 +230,7 @@ class DreamServer(BaseHTTPRequestHandler): image = self.model.sample_to_image(sample) name = f'{prefix}.{opt.seed}.{step_index}.png' metadata = f'{opt.prompt} -S{opt.seed} [intermediate]' - path = step_writer.save_image_and_prompt_to_png(image, metadata, name) + path = step_writer.save_image_and_prompt_to_png(image, dream_prompt=metadata, name=name) step_index += 1 self.wfile.write(bytes(json.dumps( {'event': 'step', 'step': step + 1, 'url': path} diff --git a/ldm/generate.py b/ldm/generate.py index 6b5cb3d794..1b3c8544e0 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -181,7 +181,7 @@ class Generate: for image, seed in results: name = f'{prefix}.{seed}.png' path = pngwriter.save_image_and_prompt_to_png( - image, f'{prompt} -S{seed}', name) + image, dream_prompt=f'{prompt} -S{seed}', name=name) outputs.append([path, seed]) return outputs diff --git a/requirements.txt b/requirements.txt index 2007ca4caf..efc55b7971 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,11 @@ test-tube torch-fidelity torchmetrics transformers +flask==2.1.3 +flask_socketio==5.3.0 +flask_cors==3.0.10 +dependency_injector==4.40.0 +eventlet git+https://github.com/openai/CLIP.git@main#egg=clip git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan diff --git a/server/application.py b/server/application.py index 2e8d77ce0f..2501f4b63d 100644 --- a/server/application.py +++ b/server/application.py @@ -7,9 +7,10 @@ import os import sys from flask import Flask from flask_cors import CORS -from flask_socketio import SocketIO, join_room, leave_room +from flask_socketio import SocketIO from omegaconf import OmegaConf from dependency_injector.wiring import inject, Provide +from ldm.dream.args import Args from server import views from server.containers import Container from server.services import GeneratorService, SignalService @@ -58,6 +59,8 @@ def run_app(config, host, port) -> Flask: # TODO: Get storage root from config app.add_url_rule('/api/images/', view_func=views.ApiImages.as_view('api_images', '../')) + app.add_url_rule('/api/images//metadata', view_func=views.ApiImagesMetadata.as_view('api_images_metadata', '../')) + app.add_url_rule('/api/images', view_func=views.ApiImagesList.as_view('api_images_list')) app.add_url_rule('/api/intermediates//', view_func=views.ApiIntermediates.as_view('api_intermediates', '../')) app.static_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '../static/dream_web/')) @@ -79,30 +82,28 @@ def run_app(config, host, port) -> Flask: def main(): """Initialize command-line parsers and the diffusion model""" - from scripts.dream import create_argv_parser - arg_parser = create_argv_parser() + arg_parser = Args() opt = arg_parser.parse_args() if opt.laion400m: - print('--laion400m flag has been deprecated. Please use --model laion400m instead.') - sys.exit(-1) - if opt.weights != 'model': - print('--weights argument has been deprecated. Please configure ./configs/models.yaml, and call it using --model instead.') - sys.exit(-1) + print('--laion400m flag has been deprecated. Please use --model laion400m instead.') + sys.exit(-1) + if opt.weights: + print('--weights argument has been deprecated. Please edit ./configs/models.yaml, and select the weights using --model instead.') + sys.exit(-1) - try: - models = OmegaConf.load(opt.config) - width = models[opt.model].width - height = models[opt.model].height - config = models[opt.model].config - weights = models[opt.model].weights - except (FileNotFoundError, IOError, KeyError) as e: - print(f'{e}. Aborting.') - sys.exit(-1) + # try: + # models = OmegaConf.load(opt.config) + # width = models[opt.model].width + # height = models[opt.model].height + # config = models[opt.model].config + # weights = models[opt.model].weights + # except (FileNotFoundError, IOError, KeyError) as e: + # print(f'{e}. Aborting.') + # sys.exit(-1) - print('* Initializing, be patient...\n') + #print('* Initializing, be patient...\n') sys.path.append('.') - from pytorch_lightning import logging # these two lines prevent a horrible warning message from appearing # when the frozen CLIP tokenizer is imported @@ -110,26 +111,28 @@ def main(): transformers.logging.set_verbosity_error() - appConfig = { - "model": { - "width": width, - "height": height, - "sampler_name": opt.sampler_name, - "weights": weights, - "full_precision": opt.full_precision, - "config": config, - "grid": opt.grid, - "latent_diffusion_weights": opt.laion400m, - "embedding_path": opt.embedding_path, - "device_type": opt.device - } - } + appConfig = opt.__dict__ + + # appConfig = { + # "model": { + # "width": width, + # "height": height, + # "sampler_name": opt.sampler_name, + # "weights": weights, + # "full_precision": opt.full_precision, + # "config": config, + # "grid": opt.grid, + # "latent_diffusion_weights": opt.laion400m, + # "embedding_path": opt.embedding_path + # } + # } # make sure the output directory exists if not os.path.exists(opt.outdir): os.makedirs(opt.outdir) # gets rid of annoying messages about random seed + from pytorch_lightning import logging logging.getLogger('pytorch_lightning').setLevel(logging.ERROR) print('\n* starting api server...') diff --git a/server/containers.py b/server/containers.py index 08ef01c4b6..a3318c5ff0 100644 --- a/server/containers.py +++ b/server/containers.py @@ -17,18 +17,24 @@ class Container(containers.DeclarativeContainer): app = None ) + # TODO: Add a model provider service that provides model(s) dynamically model_singleton = providers.ThreadSafeSingleton( Generate, - width = config.model.width, - height = config.model.height, - sampler_name = config.model.sampler_name, - weights = config.model.weights, - full_precision = config.model.full_precision, - config = config.model.config, - grid = config.model.grid, - seamless = config.model.seamless, - embedding_path = config.model.embedding_path, - device_type = config.model.device_type + model = config.model, + sampler_name = config.sampler_name, + embedding_path = config.embedding_path, + full_precision = config.full_precision + # config = config.model.config, + + # width = config.model.width, + # height = config.model.height, + # sampler_name = config.model.sampler_name, + # weights = config.model.weights, + # full_precision = config.model.full_precision, + # grid = config.model.grid, + # seamless = config.model.seamless, + # embedding_path = config.model.embedding_path, + # device_type = config.model.device_type ) # TODO: get location from config diff --git a/server/models.py b/server/models.py index 17c6d0dfe4..fc4a5f41c4 100644 --- a/server/models.py +++ b/server/models.py @@ -1,77 +1,182 @@ # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654) +from base64 import urlsafe_b64encode import json import string from copy import deepcopy from datetime import datetime, timezone from enum import Enum +from typing import Any, Dict, List, Union +from uuid import uuid4 -class DreamRequest(): - prompt: string - initimg: string - strength: float - iterations: int - steps: int - width: int - height: int - fit = None - cfgscale: float - sampler_name: string - gfpgan_strength: float - upscale_level: int - upscale_strength: float + +class DreamBase(): + # Id + id: str + + # Initial Image + enable_init_image: bool + initimg: string = None + + # Img2Img + enable_img2img: bool # TODO: support this better + strength: float = 0 # TODO: name this something related to img2img to make it clearer? + fit = None # Fit initial image dimensions + + # Generation + enable_generate: bool + prompt: string = "" + seed: int = 0 # 0 is random + steps: int = 10 + width: int = 512 + height: int = 512 + cfg_scale: float = 7.5 + sampler_name: string = 'klms' + seamless: bool = False + model: str = None # The model to use (currently unused) + embeddings = None # The embeddings to use (currently unused) + progress_images: bool = False + + # GFPGAN + enable_gfpgan: bool + gfpgan_strength: float = 0 + + # Upscale + enable_upscale: bool upscale: None - progress_images = None - seed: int + upscale_level: int = None + upscale_strength: float = 0.75 + + # Embiggen + enable_embiggen: bool + embiggen: Union[None, List[float]] = None + embiggen_tiles: Union[None, List[int]] = None + + # Metadata time: int + def __init__(self): + self.id = urlsafe_b64encode(uuid4().bytes).decode('ascii') + + def parse_json(self, j, new_instance=False): + # Id + if 'id' in j and not new_instance: + self.id = j.get('id') + + # Initial Image + self.enable_init_image = 'enable_init_image' in j and bool(j.get('enable_init_image')) + if self.enable_init_image: + self.initimg = j.get('initimg') + + # Img2Img + self.enable_img2img = 'enable_img2img' in j and bool(j.get('enable_img2img')) + if self.enable_img2img: + self.strength = float(j.get('strength')) + self.fit = 'fit' in j + + # Generation + self.enable_generate = 'enable_generate' in j and bool(j.get('enable_generate')) + if self.enable_generate: + self.prompt = j.get('prompt') + self.seed = int(j.get('seed')) + self.steps = int(j.get('steps')) + self.width = int(j.get('width')) + self.height = int(j.get('height')) + self.cfg_scale = float(j.get('cfgscale') or j.get('cfg_scale')) + self.sampler_name = j.get('sampler') or j.get('sampler_name') + # model: str = None # The model to use (currently unused) + # embeddings = None # The embeddings to use (currently unused) + self.seamless = 'seamless' in j + self.progress_images = 'progress_images' in j + + # GFPGAN + self.enable_gfpgan = 'enable_gfpgan' in j and bool(j.get('enable_gfpgan')) + if self.enable_gfpgan: + self.gfpgan_strength = float(j.get('gfpgan_strength')) + + # Upscale + self.enable_upscale = 'enable_upscale' in j and bool(j.get('enable_upscale')) + if self.enable_upscale: + self.upscale_level = j.get('upscale_level') + self.upscale_strength = j.get('upscale_strength') + self.upscale = None if self.upscale_level in {None,''} else [int(self.upscale_level),float(self.upscale_strength)] + + # Embiggen + self.enable_embiggen = 'enable_embiggen' in j and bool(j.get('enable_embiggen')) + if self.enable_embiggen: + self.embiggen = j.get('embiggen') + self.embiggen_tiles = j.get('embiggen_tiles') + + # Metadata + self.time = int(j.get('time')) if ('time' in j and not new_instance) else int(datetime.now(timezone.utc).timestamp()) + + +class DreamResult(DreamBase): + # Result + has_upscaled: False + has_gfpgan: False + # TODO: use something else for state tracking images_generated: int = 0 images_upscaled: int = 0 - def id(self, seed = None, upscaled = False) -> str: - return f"{self.time}.{seed or self.seed}{'.u' if upscaled else ''}" + def __init__(self): + super().__init__() - # TODO: handle this more cleanly (probably by splitting this into a Job and Result class) - # TODO: Set iterations to 1 or remove it from the dream result? And just keep it on the job? - def clone_without_image(self, seed = None): - data = deepcopy(self) - data.initimg = None - if seed: - data.seed = seed + def clone_without_img(self): + copy = deepcopy(self) + copy.initimg = None + return copy - return data - - def to_json(self, seed: int = None): - copy = self.clone_without_image(seed) - return json.dumps(copy.__dict__) + def to_json(self): + copy = deepcopy(self) + copy.initimg = None + j = json.dumps(copy.__dict__) + return j @staticmethod def from_json(j, newTime: bool = False): - d = DreamRequest() - d.prompt = j.get('prompt') - d.initimg = j.get('initimg') - d.strength = float(j.get('strength')) - d.iterations = int(j.get('iterations')) - d.steps = int(j.get('steps')) - d.width = int(j.get('width')) - d.height = int(j.get('height')) - d.fit = 'fit' in j - d.seamless = 'seamless' in j - d.cfgscale = float(j.get('cfgscale')) - d.sampler_name = j.get('sampler') - d.variation_amount = float(j.get('variation_amount')) - d.with_variations = j.get('with_variations') - d.gfpgan_strength = float(j.get('gfpgan_strength')) - d.upscale_level = j.get('upscale_level') - d.upscale_strength = j.get('upscale_strength') - d.upscale = [int(d.upscale_level),float(d.upscale_strength)] if d.upscale_level != '' else None - d.progress_images = 'progress_images' in j - d.seed = int(j.get('seed')) - d.time = int(datetime.now(timezone.utc).timestamp()) if newTime else int(j.get('time')) + d = DreamResult() + d.parse_json(j) return d +# TODO: switch this to a pipelined request, with pluggable steps +# Will likely require generator code changes to accomplish +class JobRequest(DreamBase): + # Iteration + iterations: int = 1 + variation_amount = None + with_variations = None + + # Results + results: List[DreamResult] = [] + + def __init__(self): + super().__init__() + + def newDreamResult(self) -> DreamResult: + result = DreamResult() + result.parse_json(self.__dict__, new_instance=True) + return result + + @staticmethod + def from_json(j): + job = JobRequest() + job.parse_json(j) + + # Metadata + job.time = int(j.get('time')) if ('time' in j) else int(datetime.now(timezone.utc).timestamp()) + + # Iteration + if job.enable_generate: + job.iterations = int(j.get('iterations')) + job.variation_amount = float(j.get('variation_amount')) + job.with_variations = j.get('with_variations') + + return job + + class ProgressType(Enum): GENERATION = 1 UPSCALING_STARTED = 2 @@ -102,11 +207,11 @@ class Signal(): # TODO: use a result id or something? Like a sub-job @staticmethod - def image_result(jobId: str, dreamId: str, dreamRequest: DreamRequest): + def image_result(jobId: str, dreamId: str, dreamResult: DreamResult): return Signal('dream_result', { 'jobId': jobId, 'dreamId': dreamId, - 'dreamRequest': dreamRequest.__dict__ + 'dreamRequest': dreamResult.clone_without_img().__dict__ }, room=jobId, broadcast=True) @staticmethod @@ -126,3 +231,21 @@ class Signal(): return Signal('job_canceled', { 'jobId': jobId }, room=jobId, broadcast=True) + + +class PaginatedItems(): + items: List[Any] + page: int # Current Page + pages: int # Total number of pages + per_page: int # Number of items per page + total: int # Total number of items in result + + def __init__(self, items: List[Any], page: int, pages: int, per_page: int, total: int): + self.items = items + self.page = page + self.pages = pages + self.per_page = per_page + self.total = total + + def to_json(self): + return json.dumps(self.__dict__) diff --git a/server/services.py b/server/services.py index 0b53cc9141..444f47cccf 100644 --- a/server/services.py +++ b/server/services.py @@ -1,25 +1,33 @@ # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654) +from argparse import ArgumentParser import base64 +from datetime import datetime, timezone +import glob +import json import os +from pathlib import Path from queue import Empty, Queue +import shlex from threading import Thread import time -from flask import app, url_for from flask_socketio import SocketIO, join_room, leave_room +from ldm.dream.args import Args +from ldm.dream.generator import embiggen +from PIL import Image from ldm.dream.pngwriter import PngWriter from ldm.dream.server import CanceledException from ldm.generate import Generate -from server.models import DreamRequest, ProgressType, Signal +from server.models import DreamResult, JobRequest, PaginatedItems, ProgressType, Signal class JobQueueService: __queue: Queue = Queue() - def push(self, dreamRequest: DreamRequest): + def push(self, dreamRequest: DreamResult): self.__queue.put(dreamRequest) - def get(self, timeout: float = None) -> DreamRequest: + def get(self, timeout: float = None) -> DreamResult: return self.__queue.get(timeout= timeout) class SignalQueueService: @@ -85,31 +93,116 @@ class LogService: self.__location = location self.__logFile = file - def log(self, dreamRequest: DreamRequest, seed = None, upscaled = False): + def log(self, dreamResult: DreamResult, seed = None, upscaled = False): with open(os.path.join(self.__location, self.__logFile), "a") as log: - log.write(f"{dreamRequest.id(seed, upscaled)}: {dreamRequest.to_json(seed)}\n") + log.write(f"{dreamResult.id}: {dreamResult.to_json()}\n") class ImageStorageService: __location: str __pngWriter: PngWriter + __legacyParser: ArgumentParser def __init__(self, location): self.__location = location self.__pngWriter = PngWriter(self.__location) + self.__legacyParser = Args() # TODO: inject this? def __getName(self, dreamId: str, postfix: str = '') -> str: return f'{dreamId}{postfix}.png' - def save(self, image, dreamRequest, seed = None, upscaled = False, postfix: str = '', metadataPostfix: str = '') -> str: - name = self.__getName(dreamRequest.id(seed, upscaled), postfix) - path = self.__pngWriter.save_image_and_prompt_to_png(image, f'{dreamRequest.prompt} -S{seed or dreamRequest.seed}{metadataPostfix}', name) + def save(self, image, dreamResult: DreamResult, postfix: str = '') -> str: + name = self.__getName(dreamResult.id, postfix) + meta = dreamResult.to_json() # TODO: make all methods consistent with writing metadata. Standardize metadata. + path = self.__pngWriter.save_image_and_prompt_to_png(image, dream_prompt=meta, metadata=None, name=name) return path def path(self, dreamId: str, postfix: str = '') -> str: name = self.__getName(dreamId, postfix) path = os.path.join(self.__location, name) return path + + # Returns true if found, false if not found or error + def delete(self, dreamId: str, postfix: str = '') -> bool: + path = self.path(dreamId, postfix) + if (os.path.exists(path)): + os.remove(path) + return True + else: + return False + + def getMetadata(self, dreamId: str, postfix: str = '') -> DreamResult: + path = self.path(dreamId, postfix) + image = Image.open(path) + text = image.text + if text.__contains__('Dream'): + dreamMeta = text.get('Dream') + try: + j = json.loads(dreamMeta) + return DreamResult.from_json(j) + except ValueError: + # Try to parse command-line format (legacy metadata format) + try: + opt = self.__parseLegacyMetadata(dreamMeta) + optd = opt.__dict__ + if (not 'width' in optd) or (optd.get('width') is None): + optd['width'] = image.width + if (not 'height' in optd) or (optd.get('height') is None): + optd['height'] = image.height + if (not 'steps' in optd) or (optd.get('steps') is None): + optd['steps'] = 10 # No way around this unfortunately - seems like it wasn't storing this previously + + optd['time'] = os.path.getmtime(path) # Set timestamp manually (won't be exactly correct though) + + return DreamResult.from_json(optd) + + except: + return None + else: + return None + + def __parseLegacyMetadata(self, command: str) -> DreamResult: + # before splitting, escape single quotes so as not to mess + # up the parser + command = command.replace("'", "\\'") + + try: + elements = shlex.split(command) + except ValueError as e: + return None + + # rearrange the arguments to mimic how it works in the Dream bot. + switches = [''] + switches_started = False + + for el in elements: + if el[0] == '-' and not switches_started: + switches_started = True + if switches_started: + switches.append(el) + else: + switches[0] += el + switches[0] += ' ' + switches[0] = switches[0][: len(switches[0]) - 1] + + try: + opt = self.__legacyParser.parse_cmd(switches) + return opt + except SystemExit: + return None + + def list_files(self, page: int, perPage: int) -> PaginatedItems: + files = sorted(glob.glob(os.path.join(self.__location,'*.png')), key=os.path.getmtime, reverse=True) + count = len(files) + + startId = page * perPage + pageCount = int(count / perPage) + 1 + endId = min(startId + perPage, count) + items = [] if startId >= count else files[startId:endId] + + items = list(map(lambda f: Path(f).stem, items)) + + return PaginatedItems(items, page, pageCount, perPage, count) class GeneratorService: @@ -144,13 +237,11 @@ class GeneratorService: # TODO: Consider moving this to its own service if there's benefit in separating the generator def __process(self): # preload the model + # TODO: support multiple models print('Preloading model') - tic = time.time() self.__model.load_model() - print( - f'>> model loaded in', '%4.2fs' % (time.time() - tic) - ) + print(f'>> model loaded in', '%4.2fs' % (time.time() - tic)) print('Started generation queue processor') try: @@ -162,103 +253,136 @@ class GeneratorService: print('Generation queue processor stopped') - def __start(self, dreamRequest: DreamRequest): - if dreamRequest.start_callback: - dreamRequest.start_callback() - self.__signal_service.emit(Signal.job_started(dreamRequest.id())) + def __on_start(self, jobRequest: JobRequest): + self.__signal_service.emit(Signal.job_started(jobRequest.id)) - def __done(self, dreamRequest: DreamRequest, image, seed, upscaled=False): - self.__imageStorage.save(image, dreamRequest, seed, upscaled) + def __on_image_result(self, jobRequest: JobRequest, image, seed, upscaled=False): + dreamResult = jobRequest.newDreamResult() + dreamResult.seed = seed + dreamResult.has_upscaled = upscaled + dreamResult.iterations = 1 + jobRequest.results.append(dreamResult) + # TODO: Separate status of GFPGAN? + + self.__imageStorage.save(image, dreamResult) - # TODO: handle upscaling logic better (this is appending data to log, but only on first generation) if not upscaled: - self.__log.log(dreamRequest, seed, upscaled) + self.__log.log(dreamResult) - self.__signal_service.emit(Signal.image_result(dreamRequest.id(), dreamRequest.id(seed, upscaled), dreamRequest.clone_without_image(seed))) + # Send result signal + self.__signal_service.emit(Signal.image_result(jobRequest.id, dreamResult.id, dreamResult)) - upscaling_requested = dreamRequest.upscale or dreamRequest.gfpgan_strength>0 + upscaling_requested = dreamResult.enable_upscale or dreamResult.enable_gfpgan - if upscaled: - dreamRequest.images_upscaled += 1 - else: - dreamRequest.images_generated +=1 - if upscaling_requested: - # action = None - if dreamRequest.images_generated >= dreamRequest.iterations: - progressType = ProgressType.UPSCALING_DONE - if dreamRequest.images_upscaled < dreamRequest.iterations: - progressType = ProgressType.UPSCALING_STARTED - self.__signal_service.emit(Signal.image_progress(dreamRequest.id(), dreamRequest.id(seed), dreamRequest.images_upscaled+1, dreamRequest.iterations, progressType)) + # Report upscaling status + # TODO: this is very coupled to logic inside the generator. Fix that. + if upscaling_requested and any(result.has_upscaled for result in jobRequest.results): + progressType = ProgressType.UPSCALING_STARTED if len(jobRequest.results) < 2 * jobRequest.iterations else ProgressType.UPSCALING_DONE + upscale_count = sum(1 for i in jobRequest.results if i.has_upscaled) + self.__signal_service.emit(Signal.image_progress(jobRequest.id, dreamResult.id, upscale_count, jobRequest.iterations, progressType)) - def __progress(self, dreamRequest, sample, step): + def __on_progress(self, jobRequest: JobRequest, sample, step): if self.__cancellationRequested: self.__cancellationRequested = False raise CanceledException + # TODO: Progress per request will be easier once the seeds (and ids) can all be pre-generated hasProgressImage = False - if dreamRequest.progress_images and step % 5 == 0 and step < dreamRequest.steps - 1: + s = str(len(jobRequest.results)) + if jobRequest.progress_images and step % 5 == 0 and step < jobRequest.steps - 1: image = self.__model._sample_to_image(sample) - self.__intermediateStorage.save(image, dreamRequest, self.__model.seed, postfix=f'.{step}', metadataPostfix=f' [intermediate]') + + # TODO: clean this up, use a pre-defined dream result + result = DreamResult() + result.parse_json(jobRequest.__dict__, new_instance=False) + self.__intermediateStorage.save(image, result, postfix=f'.{s}.{step}') hasProgressImage = True - self.__signal_service.emit(Signal.image_progress(dreamRequest.id(), dreamRequest.id(self.__model.seed), step, dreamRequest.steps, ProgressType.GENERATION, hasProgressImage)) + self.__signal_service.emit(Signal.image_progress(jobRequest.id, f'{jobRequest.id}.{s}', step, jobRequest.steps, ProgressType.GENERATION, hasProgressImage)) - def __generate(self, dreamRequest: DreamRequest): + def __generate(self, jobRequest: JobRequest): try: - initimgfile = None - if dreamRequest.initimg is not None: - with open("./img2img-tmp.png", "wb") as f: - initimg = dreamRequest.initimg.split(",")[1] # Ignore mime type - f.write(base64.b64decode(initimg)) - initimgfile = "./img2img-tmp.png" + # TODO: handle this file a file service for init images + initimgfile = None # TODO: support this on the model directly? + if (jobRequest.enable_init_image): + if jobRequest.initimg is not None: + with open("./img2img-tmp.png", "wb") as f: + initimg = jobRequest.initimg.split(",")[1] # Ignore mime type + f.write(base64.b64decode(initimg)) + initimgfile = "./img2img-tmp.png" - # Get a random seed if we don't have one yet - # TODO: handle "previous" seed usage? - if dreamRequest.seed == -1: - dreamRequest.seed = self.__model.seed + # Use previous seed if set to -1 + initSeed = jobRequest.seed + if initSeed == -1: + initSeed = self.__model.seed # Zero gfpgan strength if the model doesn't exist # TODO: determine if this could be at the top now? Used to cause circular import from ldm.gfpgan.gfpgan_tools import gfpgan_model_exists if not gfpgan_model_exists: - dreamRequest.gfpgan_strength = 0 + jobRequest.enable_gfpgan = False - self.__start(dreamRequest) + # Signal start + self.__on_start(jobRequest) - self.__model.prompt2image( - prompt = dreamRequest.prompt, - init_img = initimgfile, # TODO: ensure this works - strength = None if initimgfile is None else dreamRequest.strength, - fit = None if initimgfile is None else dreamRequest.fit, - iterations = dreamRequest.iterations, - cfg_scale = dreamRequest.cfgscale, - width = dreamRequest.width, - height = dreamRequest.height, - seed = dreamRequest.seed, - steps = dreamRequest.steps, - variation_amount = dreamRequest.variation_amount, - with_variations = dreamRequest.with_variations, - gfpgan_strength = dreamRequest.gfpgan_strength, - upscale = dreamRequest.upscale, - sampler_name = dreamRequest.sampler_name, - seamless = dreamRequest.seamless, - step_callback = lambda sample, step: self.__progress(dreamRequest, sample, step), - image_callback = lambda image, seed, upscaled=False: self.__done(dreamRequest, image, seed, upscaled)) + # Generate in model + # TODO: Split job generation requests instead of fitting all parameters here + # TODO: Support no generation (just upscaling/gfpgan) + + upscale = None if not jobRequest.enable_upscale else jobRequest.upscale + gfpgan_strength = 0 if not jobRequest.enable_gfpgan else jobRequest.gfpgan_strength + + if not jobRequest.enable_generate: + # If not generating, check if we're upscaling or running gfpgan + if not upscale and not gfpgan_strength: + # Invalid settings (TODO: Add message to help user) + raise CanceledException() + + image = Image.open(initimgfile) + # TODO: support progress for upscale? + self.__model.upscale_and_reconstruct( + image_list = [[image,0]], + upscale = upscale, + strength = gfpgan_strength, + save_original = False, + image_callback = lambda image, seed, upscaled=False: self.__on_image_result(jobRequest, image, seed, upscaled)) + + else: + # Generating - run the generation + init_img = None if (not jobRequest.enable_img2img or jobRequest.strength == 0) else initimgfile + + + self.__model.prompt2image( + prompt = jobRequest.prompt, + init_img = init_img, # TODO: ensure this works + strength = None if init_img is None else jobRequest.strength, + fit = None if init_img is None else jobRequest.fit, + iterations = jobRequest.iterations, + cfg_scale = jobRequest.cfg_scale, + width = jobRequest.width, + height = jobRequest.height, + seed = jobRequest.seed, + steps = jobRequest.steps, + variation_amount = jobRequest.variation_amount, + with_variations = jobRequest.with_variations, + gfpgan_strength = gfpgan_strength, + upscale = upscale, + sampler_name = jobRequest.sampler_name, + seamless = jobRequest.seamless, + embiggen = jobRequest.embiggen, + embiggen_tiles = jobRequest.embiggen_tiles, + step_callback = lambda sample, step: self.__on_progress(jobRequest, sample, step), + image_callback = lambda image, seed, upscaled=False: self.__on_image_result(jobRequest, image, seed, upscaled)) except CanceledException: - if dreamRequest.cancelled_callback: - dreamRequest.cancelled_callback() - - self.__signal_service.emit(Signal.job_canceled(dreamRequest.id())) + self.__signal_service.emit(Signal.job_canceled(jobRequest.id)) finally: - if dreamRequest.done_callback: - dreamRequest.done_callback() - self.__signal_service.emit(Signal.job_done(dreamRequest.id())) + self.__signal_service.emit(Signal.job_done(jobRequest.id)) # Remove the temp file if (initimgfile is not None): diff --git a/server/views.py b/server/views.py index 590adc6532..db4857d14f 100644 --- a/server/views.py +++ b/server/views.py @@ -8,7 +8,7 @@ from flask import current_app, jsonify, request, Response, send_from_directory, from flask.views import MethodView from dependency_injector.wiring import inject, Provide -from server.models import DreamRequest +from server.models import DreamResult, JobRequest from server.services import GeneratorService, ImageStorageService, JobQueueService from server.containers import Container @@ -16,23 +16,14 @@ class ApiJobs(MethodView): @inject def post(self, job_queue_service: JobQueueService = Provide[Container.generation_queue_service]): - dreamRequest = DreamRequest.from_json(request.json, newTime = True) + jobRequest = JobRequest.from_json(request.json) - #self.canceled.clear() - print(f">> Request to generate with prompt: {dreamRequest.prompt}") - - q = Queue() - - dreamRequest.start_callback = None - dreamRequest.image_callback = None - dreamRequest.progress_callback = None - dreamRequest.cancelled_callback = None - dreamRequest.done_callback = None + print(f">> Request to generate with prompt: {jobRequest.prompt}") # Push the request - job_queue_service.push(dreamRequest) + job_queue_service.push(jobRequest) - return { 'dreamId': dreamRequest.id() } + return { 'jobId': jobRequest.id } class WebIndex(MethodView): @@ -68,6 +59,7 @@ class ApiCancel(MethodView): return Response(status=204) +# TODO: Combine all image storage access class ApiImages(MethodView): init_every_request = False __pathRoot = None @@ -82,6 +74,27 @@ class ApiImages(MethodView): name = self.__storage.path(dreamId) fullpath=os.path.join(self.__pathRoot, name) return send_from_directory(os.path.dirname(fullpath), os.path.basename(fullpath)) + + def delete(self, dreamId): + result = self.__storage.delete(dreamId) + return Response(status=204) if result else Response(status=404) + + +class ApiImagesMetadata(MethodView): + init_every_request = False + __pathRoot = None + __storage: ImageStorageService + + @inject + def __init__(self, pathBase, storage: ImageStorageService = Provide[Container.image_storage_service]): + self.__pathRoot = os.path.abspath(os.path.join(os.path.dirname(__file__), pathBase)) + self.__storage = storage + + def get(self, dreamId): + meta = self.__storage.getMetadata(dreamId) + j = {} if meta is None else meta.__dict__ + return j + class ApiIntermediates(MethodView): init_every_request = False @@ -97,3 +110,23 @@ class ApiIntermediates(MethodView): name = self.__storage.path(dreamId, postfix=f'.{step}') fullpath=os.path.join(self.__pathRoot, name) return send_from_directory(os.path.dirname(fullpath), os.path.basename(fullpath)) + + def delete(self, dreamId): + result = self.__storage.delete(dreamId) + return Response(status=204) if result else Response(status=404) + + +class ApiImagesList(MethodView): + init_every_request = False + __storage: ImageStorageService + + @inject + def __init__(self, storage: ImageStorageService = Provide[Container.image_storage_service]): + self.__storage = storage + + def get(self): + page = request.args.get("page", default=0, type=int) + perPage = request.args.get("per_page", default=10, type=int) + + result = self.__storage.list_files(page, perPage) + return result.__dict__ diff --git a/static/dream_web/index.css b/static/dream_web/index.css index 51f0f267c3..25a0994a3d 100644 --- a/static/dream_web/index.css +++ b/static/dream_web/index.css @@ -1,3 +1,8 @@ +:root { + --fields-dark:#DCDCDC; + --fields-light:#F5F5F5; +} + * { font-family: 'Arial'; font-size: 100%; @@ -18,15 +23,26 @@ fieldset { border: none; line-height: 2.2em; } +fieldset > legend { + width: auto; + margin-left: 0; + margin-right: auto; + font-weight:bold; +} select, input { margin-right: 10px; padding: 2px; } +input:disabled { + cursor:auto; +} input[type=submit] { + cursor: pointer; background-color: #666; color: white; } input[type=checkbox] { + cursor: pointer; margin-right: 0px; width: 20px; height: 20px; @@ -87,11 +103,11 @@ header h1 { } #results img { border-radius: 5px; - object-fit: cover; + object-fit: contain; + background-color: var(--fields-dark); } #fieldset-config { line-height:2em; - background-color: #F0F0F0; } input[type="number"] { width: 60px; @@ -118,35 +134,46 @@ label { #progress-image { width: 30vh; height: 30vh; + object-fit: contain; + background-color: var(--fields-dark); } #cancel-button { cursor: pointer; color: red; } -#basic-parameters { - background-color: #EEEEEE; -} #txt2img { - background-color: #DCDCDC; + background-color: var(--fields-dark); } #variations { - background-color: #EEEEEE; + background-color: var(--fields-light); +} +#initimg { + background-color: var(--fields-dark); } #img2img { - background-color: #DCDCDC; + background-color: var(--fields-light); } -#gfpgan { - background-color: #EEEEEE; +#initimg > :not(legend) { + background-color: var(--fields-light); + margin: .5em; +} + +#postprocess, #initimg { + display:flex; + flex-wrap:wrap; + padding: 0; + margin-top: 1em; + background-color: var(--fields-dark); +} +#postprocess > fieldset, #initimg > * { + flex-grow: 1; +} +#postprocess > fieldset { + background-color: var(--fields-dark); } #progress-section { - background-color: #F5F5F5; -} -.section-header { - text-align: left; - font-weight: bold; - padding: 0 0 0 0; + background-color: var(--fields-light); } #no-results-message:not(:only-child) { display: none; } - diff --git a/static/dream_web/index.html b/static/dream_web/index.html index b8c80fe838..9dbd213669 100644 --- a/static/dream_web/index.html +++ b/static/dream_web/index.html @@ -1,104 +1,152 @@ - - Stable Diffusion Dream Server - - - - - - - - - -
-

Stable Diffusion Dream Server

-
- For news and support for this web service, visit our GitHub site + + Stable Diffusion Dream Server + + + + + + + + + + + +
+

Stable Diffusion Dream Server

+
+ For news and support for this web service, visit our GitHub + site +
+
+ +
+ +
+
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+ + + + +
+
+
+ + + + +
-
- - - - -
+
+
+ + + + + + + + +
+
+
-
Post-processing options
- - - + + + + + + +
+
+ + + + +
- -
-
-
- - -
- -
- Postprocessing...1/3 -
- -
- -
-
-

No results...

+
+ + +
+
+
+ + +
+ +
+ Postprocessing...1/3
- - +
+ +
+
+ + + diff --git a/static/dream_web/index.js b/static/dream_web/index.js index 3af0308fb5..5de690297d 100644 --- a/static/dream_web/index.js +++ b/static/dream_web/index.js @@ -1,5 +1,73 @@ const socket = io(); +var priorResultsLoadState = { + page: 0, + pages: 1, + per_page: 10, + total: 20, + offset: 0, // number of items generated since last load + loading: false, + initialized: false +}; + +function loadPriorResults() { + // Fix next page by offset + let offsetPages = priorResultsLoadState.offset / priorResultsLoadState.per_page; + priorResultsLoadState.page += offsetPages; + priorResultsLoadState.pages += offsetPages; + priorResultsLoadState.total += priorResultsLoadState.offset; + priorResultsLoadState.offset = 0; + + if (priorResultsLoadState.loading) { + return; + } + + if (priorResultsLoadState.page >= priorResultsLoadState.pages) { + return; // Nothing more to load + } + + // Load + priorResultsLoadState.loading = true + let url = new URL('/api/images', document.baseURI); + url.searchParams.append('page', priorResultsLoadState.initialized ? priorResultsLoadState.page + 1 : priorResultsLoadState.page); + url.searchParams.append('per_page', priorResultsLoadState.per_page); + fetch(url.href, { + method: 'GET', + headers: new Headers({'content-type': 'application/json'}) + }) + .then(response => response.json()) + .then(data => { + priorResultsLoadState.page = data.page; + priorResultsLoadState.pages = data.pages; + priorResultsLoadState.per_page = data.per_page; + priorResultsLoadState.total = data.total; + + data.items.forEach(function(dreamId, index) { + let src = 'api/images/' + dreamId; + fetch('/api/images/' + dreamId + '/metadata', { + method: 'GET', + headers: new Headers({'content-type': 'application/json'}) + }) + .then(response => response.json()) + .then(metadata => { + let seed = metadata.seed || 0; // TODO: Parse old metadata + appendOutput(src, seed, metadata, true); + }); + }); + + // Load until page is full + if (!priorResultsLoadState.initialized) { + if (document.body.scrollHeight <= window.innerHeight) { + loadPriorResults(); + } + } + }) + .finally(() => { + priorResultsLoadState.loading = false; + priorResultsLoadState.initialized = true; + }); +} + function resetForm() { var form = document.getElementById('generate-form'); form.querySelector('fieldset').removeAttribute('disabled'); @@ -45,48 +113,64 @@ function toBase64(file) { }); } -function appendOutput(src, seed, config) { +function ondragdream(event) { + let dream = event.target.dataset.dream; + event.dataTransfer.setData("dream", dream); +} + +function seedClick(event) { + // Get element + var image = event.target.closest('figure').querySelector('img'); + var dream = JSON.parse(decodeURIComponent(image.dataset.dream)); + + let form = document.querySelector("#generate-form"); + for (const [k, v] of new FormData(form)) { + if (k == 'initimg') { continue; } + let formElem = form.querySelector(`*[name=${k}]`); + formElem.value = dream[k] !== undefined ? dream[k] : formElem.defaultValue; + } + + document.querySelector("#seed").value = dream.seed; + document.querySelector('#iterations').value = 1; // Reset to 1 iteration since we clicked a single image (not a full job) + + // NOTE: leaving this manual for the user for now - it was very confusing with this behavior + // document.querySelector("#with_variations").value = variations || ''; + // if (document.querySelector("#variation_amount").value <= 0) { + // document.querySelector("#variation_amount").value = 0.2; + // } + + saveFields(document.querySelector("#generate-form")); +} + +function appendOutput(src, seed, config, toEnd=false) { let outputNode = document.createElement("figure"); let altText = seed.toString() + " | " + config.prompt; + // img needs width and height for lazy loading to work + // TODO: store the full config in a data attribute on the image? const figureContents = ` - ${altText} + ${altText} -
${seed}
+
${seed}
`; outputNode.innerHTML = figureContents; - let figcaption = outputNode.querySelector('figcaption') - // Reload image config - figcaption.addEventListener('click', () => { - let form = document.querySelector("#generate-form"); - for (const [k, v] of new FormData(form)) { - if (k == 'initimg') { continue; } - form.querySelector(`*[name=${k}]`).value = config[k]; - } - if (config.variation_amount > 0 || config.with_variations != '') { - document.querySelector("#seed").value = config.seed; - } else { - document.querySelector("#seed").value = seed; - } - - if (config.variation_amount > 0) { - let oldVarAmt = document.querySelector("#variation_amount").value - let oldVariations = document.querySelector("#with_variations").value - let varSep = '' - document.querySelector("#variation_amount").value = 0; - if (document.querySelector("#with_variations").value != '') { - varSep = "," - } - document.querySelector("#with_variations").value = oldVariations + varSep + seed + ':' + config.variation_amount - } - - saveFields(document.querySelector("#generate-form")); - }); - - document.querySelector("#results").prepend(outputNode); + if (toEnd) { + document.querySelector("#results").append(outputNode); + } else { + document.querySelector("#results").prepend(outputNode); + } document.querySelector("#no-results-message")?.remove(); } @@ -119,14 +203,33 @@ async function generateSubmit(form) { // Convert file data to base64 // TODO: Should probably uplaod files with formdata or something, and store them in the backend? let formData = Object.fromEntries(new FormData(form)); + if (!formData.enable_generate && !formData.enable_init_image) { + gen_label = document.querySelector("label[for=enable_generate]").innerHTML; + initimg_label = document.querySelector("label[for=enable_init_image]").innerHTML; + alert(`Error: one of "${gen_label}" or "${initimg_label}" must be set`); + } + + formData.initimg_name = formData.initimg.name formData.initimg = formData.initimg.name !== '' ? await toBase64(formData.initimg) : null; + // Evaluate all checkboxes + let checkboxes = form.querySelectorAll('input[type=checkbox]'); + checkboxes.forEach(function (checkbox) { + if (checkbox.checked) { + formData[checkbox.name] = 'true'; + } + }); + let strength = formData.strength; let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; + let showProgressImages = formData.progress_images; + + // Set enabling flags + // Initialize the progress bar - initProgress(totalSteps); + initProgress(totalSteps, showProgressImages); // POST, use response to listen for events fetch(form.action, { @@ -136,13 +239,19 @@ async function generateSubmit(form) { }) .then(response => response.json()) .then(data => { - var dreamId = data.dreamId; - socket.emit('join_room', { 'room': dreamId }); + var jobId = data.jobId; + socket.emit('join_room', { 'room': jobId }); }); form.querySelector('fieldset').setAttribute('disabled',''); } +function fieldSetEnableChecked(event) { + cb = event.target; + fields = cb.closest('fieldset'); + fields.disabled = !cb.checked; +} + // Socket listeners socket.on('job_started', (data) => {}) @@ -152,6 +261,7 @@ socket.on('dream_result', (data) => { var dreamRequest = data.dreamRequest; var src = 'api/images/' + dreamId; + priorResultsLoadState.offset += 1; appendOutput(src, dreamRequest.seed, dreamRequest); resetProgress(false); @@ -193,7 +303,13 @@ socket.on('job_done', (data) => { resetProgress(); }) -window.onload = () => { +window.onload = async () => { + document.querySelector("#prompt").addEventListener("keydown", (e) => { + if (e.key === "Enter" && !e.shiftKey) { + const form = e.target.form; + generateSubmit(form); + } + }); document.querySelector("#generate-form").addEventListener('submit', (e) => { e.preventDefault(); const form = e.target; @@ -216,12 +332,65 @@ window.onload = () => { loadFields(document.querySelector("#generate-form")); document.querySelector('#cancel-button').addEventListener('click', () => { - fetch('/cancel').catch(e => { + fetch('/api/cancel').catch(e => { console.error(e); }); }); + document.documentElement.addEventListener('keydown', (e) => { + if (e.key === "Escape") + fetch('/api/cancel').catch(err => { + console.error(err); + }); + }); if (!config.gfpgan_model_exists) { document.querySelector("#gfpgan").style.display = 'none'; } + + window.addEventListener("scroll", () => { + if ((window.innerHeight + window.pageYOffset) >= document.body.offsetHeight) { + loadPriorResults(); + } + }); + + + + // Enable/disable forms by checkboxes + document.querySelectorAll("legend > input[type=checkbox]").forEach(function(cb) { + cb.addEventListener('change', fieldSetEnableChecked); + fieldSetEnableChecked({ target: cb}) + }); + + + // Load some of the previous results + loadPriorResults(); + + // Image drop/upload WIP + /* + let drop = document.getElementById('dropper'); + function ondrop(event) { + let dreamData = event.dataTransfer.getData('dream'); + if (dreamData) { + var dream = JSON.parse(decodeURIComponent(dreamData)); + alert(dream.dreamId); + } + }; + + function ondragenter(event) { + event.preventDefault(); + }; + + function ondragover(event) { + event.preventDefault(); + }; + + function ondragleave(event) { + + } + + drop.addEventListener('drop', ondrop); + drop.addEventListener('dragenter', ondragenter); + drop.addEventListener('dragover', ondragover); + drop.addEventListener('dragleave', ondragleave); + */ }; From d1de1e357a889a350d682b03842ffca633bbf359 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat, 17 Sep 2022 06:15:55 +1000 Subject: [PATCH 068/238] Fixes PromptFormatter import bug --- backend/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/server.py b/backend/server.py index 8dbbc65cb1..02fc0b487f 100644 --- a/backend/server.py +++ b/backend/server.py @@ -19,7 +19,8 @@ from uuid import uuid4 from ldm.gfpgan.gfpgan_tools import real_esrgan_upscale from ldm.gfpgan.gfpgan_tools import run_gfpgan from ldm.generate import Generate -from ldm.dream.pngwriter import PngWriter +from ldm.dream.pngwriter import PngWriter, retrieve_metadata + from modules.parameters import parameters_to_command, create_cmd_parser From f9feaac8c728162f43f917364b96203fee8071c9 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat, 17 Sep 2022 06:16:16 +1000 Subject: [PATCH 069/238] Fixes metadata related to new args --- backend/server.py | 17 +++++++---------- ldm/dream/pngwriter.py | 8 +++++--- scripts/sd-metadata.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/backend/server.py b/backend/server.py index 02fc0b487f..3d24656af0 100644 --- a/backend/server.py +++ b/backend/server.py @@ -147,15 +147,12 @@ def handle_request_all_images(): paths.sort(key=lambda x: os.path.getmtime(x)) image_array = [] for path in paths: - image = Image.open(path) - metadata = {} - if 'Dream' in image.info: - try: - metadata = vars(parser.parse_args(shlex.split(image.info['Dream']))) - except SystemExit: - # TODO: Unable to parse metadata, ignore it for now, - # this can happen when metadata is missing a prompt - pass + # image = Image.open(path) + all_metadata = retrieve_metadata(path) + if 'Dream' in all_metadata and not all_metadata['sd-metadata']: + metadata = vars(parser.parse_args(shlex.split(all_metadata['Dream']))) + else: + metadata = all_metadata['sd-metadata'] image_array.append({'path': path, 'metadata': metadata}) return make_response("OK", data=image_array) @@ -308,7 +305,7 @@ def save_image(image, parameters, output_dir, step_index=None, postprocessing=Fa command = parameters_to_command(parameters) - path = pngwriter.save_image_and_prompt_to_png(image, command, filename) + path = pngwriter.save_image_and_prompt_to_png(image, command, parameters, filename) return path diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index a8d2425b91..ecbc3c0e15 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -47,7 +47,8 @@ class PngWriter: metadata stored there, as a dict ''' path = os.path.join(self.outdir,img_basename) - return retrieve_metadata(path) + all_metadata = retrieve_metadata(path) + return all_metadata['sd-metadata'] def retrieve_metadata(img_path): ''' @@ -55,6 +56,7 @@ def retrieve_metadata(img_path): metadata stored there, as a dict ''' im = Image.open(img_path) - md = im.text.get('sd-metadata',{}) - return json.loads(md) + md = im.text.get('sd-metadata', '{}') + dream_prompt = im.text.get('Dream', '') + return {'sd-metadata': json.loads(md), 'Dream': dream_prompt} diff --git a/scripts/sd-metadata.py b/scripts/sd-metadata.py index a3438fa078..02d5002d60 100644 --- a/scripts/sd-metadata.py +++ b/scripts/sd-metadata.py @@ -13,7 +13,7 @@ filenames = sys.argv[1:] for f in filenames: try: metadata = retrieve_metadata(f) - print(f'{f}:\n',json.dumps(metadata, indent=4)) + print(f'{f}:\n',json.dumps(metadata['sd-metadata'], indent=4)) except FileNotFoundError: sys.stderr.write(f'{f} not found\n') continue From 67fbaa7c3106ee1c6461048cbff9111df5af0abc Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 16:57:54 -0400 Subject: [PATCH 070/238] reconciled conflicting changes to pngwriter call --- backend/server.py | 2 +- ldm/dream/pngwriter.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/server.py b/backend/server.py index 3d24656af0..ef93c5b0d7 100644 --- a/backend/server.py +++ b/backend/server.py @@ -305,7 +305,7 @@ def save_image(image, parameters, output_dir, step_index=None, postprocessing=Fa command = parameters_to_command(parameters) - path = pngwriter.save_image_and_prompt_to_png(image, command, parameters, filename) + path = pngwriter.save_image_and_prompt_to_png(image, command, metadata=parameters, name=filename) return path diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index 5cda259357..9a2a8bc816 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -34,6 +34,7 @@ class PngWriter: # saves image named _image_ to outdir/name, writing metadata from prompt # returns full path of output def save_image_and_prompt_to_png(self, image, dream_prompt, name, metadata=None): + print(f'self.outdir={self.outdir}, name={name}') path = os.path.join(self.outdir, name) info = PngImagePlugin.PngInfo() info.add_text('Dream', dream_prompt) From fe12c6c099ddbc1939a06d37b9b63ed58566ef15 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 16:58:16 -0400 Subject: [PATCH 071/238] Squashed commit of the following: commit 67fbaa7c3106ee1c6461048cbff9111df5af0abc Author: Lincoln Stein Date: Fri Sep 16 16:57:54 2022 -0400 reconciled conflicting changes to pngwriter call commit ddc68b01f7a50901ef8d7ceb250ce4a337762819 Merge: f9feaac cbac95b Author: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat Sep 17 06:39:22 2022 +1000 Merge remote-tracking branch 'upstream/development' into development commit f9feaac8c728162f43f917364b96203fee8071c9 Author: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat Sep 17 06:16:16 2022 +1000 Fixes metadata related to new args commit d1de1e357a889a350d682b03842ffca633bbf359 Author: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat Sep 17 06:15:55 2022 +1000 Fixes PromptFormatter import bug --- backend/server.py | 20 +++++++++----------- ldm/dream/pngwriter.py | 9 ++++++--- scripts/sd-metadata.py | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/backend/server.py b/backend/server.py index 8dbbc65cb1..ef93c5b0d7 100644 --- a/backend/server.py +++ b/backend/server.py @@ -19,7 +19,8 @@ from uuid import uuid4 from ldm.gfpgan.gfpgan_tools import real_esrgan_upscale from ldm.gfpgan.gfpgan_tools import run_gfpgan from ldm.generate import Generate -from ldm.dream.pngwriter import PngWriter +from ldm.dream.pngwriter import PngWriter, retrieve_metadata + from modules.parameters import parameters_to_command, create_cmd_parser @@ -146,15 +147,12 @@ def handle_request_all_images(): paths.sort(key=lambda x: os.path.getmtime(x)) image_array = [] for path in paths: - image = Image.open(path) - metadata = {} - if 'Dream' in image.info: - try: - metadata = vars(parser.parse_args(shlex.split(image.info['Dream']))) - except SystemExit: - # TODO: Unable to parse metadata, ignore it for now, - # this can happen when metadata is missing a prompt - pass + # image = Image.open(path) + all_metadata = retrieve_metadata(path) + if 'Dream' in all_metadata and not all_metadata['sd-metadata']: + metadata = vars(parser.parse_args(shlex.split(all_metadata['Dream']))) + else: + metadata = all_metadata['sd-metadata'] image_array.append({'path': path, 'metadata': metadata}) return make_response("OK", data=image_array) @@ -307,7 +305,7 @@ def save_image(image, parameters, output_dir, step_index=None, postprocessing=Fa command = parameters_to_command(parameters) - path = pngwriter.save_image_and_prompt_to_png(image, command, filename) + path = pngwriter.save_image_and_prompt_to_png(image, command, metadata=parameters, name=filename) return path diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index 6291059585..9a2a8bc816 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -34,6 +34,7 @@ class PngWriter: # saves image named _image_ to outdir/name, writing metadata from prompt # returns full path of output def save_image_and_prompt_to_png(self, image, dream_prompt, name, metadata=None): + print(f'self.outdir={self.outdir}, name={name}') path = os.path.join(self.outdir, name) info = PngImagePlugin.PngInfo() info.add_text('Dream', dream_prompt) @@ -48,7 +49,8 @@ class PngWriter: metadata stored there, as a dict ''' path = os.path.join(self.outdir,img_basename) - return retrieve_metadata(path) + all_metadata = retrieve_metadata(path) + return all_metadata['sd-metadata'] def retrieve_metadata(img_path): ''' @@ -56,6 +58,7 @@ def retrieve_metadata(img_path): metadata stored there, as a dict ''' im = Image.open(img_path) - md = im.text.get('sd-metadata',{}) - return json.loads(md) + md = im.text.get('sd-metadata', '{}') + dream_prompt = im.text.get('Dream', '') + return {'sd-metadata': json.loads(md), 'Dream': dream_prompt} diff --git a/scripts/sd-metadata.py b/scripts/sd-metadata.py index a3438fa078..02d5002d60 100644 --- a/scripts/sd-metadata.py +++ b/scripts/sd-metadata.py @@ -13,7 +13,7 @@ filenames = sys.argv[1:] for f in filenames: try: metadata = retrieve_metadata(f) - print(f'{f}:\n',json.dumps(metadata, indent=4)) + print(f'{f}:\n',json.dumps(metadata['sd-metadata'], indent=4)) except FileNotFoundError: sys.stderr.write(f'{f} not found\n') continue From 6cb6c4a9114d8f6420fbc674f37fb22806a263be Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 17:27:08 -0400 Subject: [PATCH 072/238] restore static files for old web server --- static/dream_web/index.css | 61 ++---- static/dream_web/index.html | 274 +++++++++++--------------- static/dream_web/index.js | 383 ++++++++++-------------------------- 3 files changed, 229 insertions(+), 489 deletions(-) diff --git a/static/dream_web/index.css b/static/dream_web/index.css index 25a0994a3d..51f0f267c3 100644 --- a/static/dream_web/index.css +++ b/static/dream_web/index.css @@ -1,8 +1,3 @@ -:root { - --fields-dark:#DCDCDC; - --fields-light:#F5F5F5; -} - * { font-family: 'Arial'; font-size: 100%; @@ -23,26 +18,15 @@ fieldset { border: none; line-height: 2.2em; } -fieldset > legend { - width: auto; - margin-left: 0; - margin-right: auto; - font-weight:bold; -} select, input { margin-right: 10px; padding: 2px; } -input:disabled { - cursor:auto; -} input[type=submit] { - cursor: pointer; background-color: #666; color: white; } input[type=checkbox] { - cursor: pointer; margin-right: 0px; width: 20px; height: 20px; @@ -103,11 +87,11 @@ header h1 { } #results img { border-radius: 5px; - object-fit: contain; - background-color: var(--fields-dark); + object-fit: cover; } #fieldset-config { line-height:2em; + background-color: #F0F0F0; } input[type="number"] { width: 60px; @@ -134,46 +118,35 @@ label { #progress-image { width: 30vh; height: 30vh; - object-fit: contain; - background-color: var(--fields-dark); } #cancel-button { cursor: pointer; color: red; } +#basic-parameters { + background-color: #EEEEEE; +} #txt2img { - background-color: var(--fields-dark); + background-color: #DCDCDC; } #variations { - background-color: var(--fields-light); -} -#initimg { - background-color: var(--fields-dark); + background-color: #EEEEEE; } #img2img { - background-color: var(--fields-light); + background-color: #DCDCDC; } -#initimg > :not(legend) { - background-color: var(--fields-light); - margin: .5em; -} - -#postprocess, #initimg { - display:flex; - flex-wrap:wrap; - padding: 0; - margin-top: 1em; - background-color: var(--fields-dark); -} -#postprocess > fieldset, #initimg > * { - flex-grow: 1; -} -#postprocess > fieldset { - background-color: var(--fields-dark); +#gfpgan { + background-color: #EEEEEE; } #progress-section { - background-color: var(--fields-light); + background-color: #F5F5F5; +} +.section-header { + text-align: left; + font-weight: bold; + padding: 0 0 0 0; } #no-results-message:not(:only-child) { display: none; } + diff --git a/static/dream_web/index.html b/static/dream_web/index.html index 9dbd213669..1e194c0205 100644 --- a/static/dream_web/index.html +++ b/static/dream_web/index.html @@ -1,152 +1,102 @@ - - - Stable Diffusion Dream Server - - - - - - - - - - - -
-

Stable Diffusion Dream Server

-
- For news and support for this web service, visit our GitHub - site -
-
- -
- -
-
- - - - - - - - - - - - - - - -
- - - - - - - - - - -
- - - - -
-
-
- - - - -
+ + +
+ +
+ +
+
+
Basic options
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+
+
Image-to-image options
-
-
- - - - - - - - -
-
-
+
+ + + + +
- - - - - - -
-
- - - - - +
Post-processing options
+ + +
-
- -
-
-
-
- - -
- -
- Postprocessing...1/3 + +
+
+
+ + +
+ +
+ Postprocessing...1/3 +
+ +
+ +
+
+

No results...

-
- -
-
-
- - + + diff --git a/static/dream_web/index.js b/static/dream_web/index.js index 5de690297d..ac68034920 100644 --- a/static/dream_web/index.js +++ b/static/dream_web/index.js @@ -1,109 +1,3 @@ -const socket = io(); - -var priorResultsLoadState = { - page: 0, - pages: 1, - per_page: 10, - total: 20, - offset: 0, // number of items generated since last load - loading: false, - initialized: false -}; - -function loadPriorResults() { - // Fix next page by offset - let offsetPages = priorResultsLoadState.offset / priorResultsLoadState.per_page; - priorResultsLoadState.page += offsetPages; - priorResultsLoadState.pages += offsetPages; - priorResultsLoadState.total += priorResultsLoadState.offset; - priorResultsLoadState.offset = 0; - - if (priorResultsLoadState.loading) { - return; - } - - if (priorResultsLoadState.page >= priorResultsLoadState.pages) { - return; // Nothing more to load - } - - // Load - priorResultsLoadState.loading = true - let url = new URL('/api/images', document.baseURI); - url.searchParams.append('page', priorResultsLoadState.initialized ? priorResultsLoadState.page + 1 : priorResultsLoadState.page); - url.searchParams.append('per_page', priorResultsLoadState.per_page); - fetch(url.href, { - method: 'GET', - headers: new Headers({'content-type': 'application/json'}) - }) - .then(response => response.json()) - .then(data => { - priorResultsLoadState.page = data.page; - priorResultsLoadState.pages = data.pages; - priorResultsLoadState.per_page = data.per_page; - priorResultsLoadState.total = data.total; - - data.items.forEach(function(dreamId, index) { - let src = 'api/images/' + dreamId; - fetch('/api/images/' + dreamId + '/metadata', { - method: 'GET', - headers: new Headers({'content-type': 'application/json'}) - }) - .then(response => response.json()) - .then(metadata => { - let seed = metadata.seed || 0; // TODO: Parse old metadata - appendOutput(src, seed, metadata, true); - }); - }); - - // Load until page is full - if (!priorResultsLoadState.initialized) { - if (document.body.scrollHeight <= window.innerHeight) { - loadPriorResults(); - } - } - }) - .finally(() => { - priorResultsLoadState.loading = false; - priorResultsLoadState.initialized = true; - }); -} - -function resetForm() { - var form = document.getElementById('generate-form'); - form.querySelector('fieldset').removeAttribute('disabled'); -} - -function initProgress(totalSteps, showProgressImages) { - // TODO: Progress could theoretically come from multiple jobs at the same time (in the future) - let progressSectionEle = document.querySelector('#progress-section'); - progressSectionEle.style.display = 'initial'; - let progressEle = document.querySelector('#progress-bar'); - progressEle.setAttribute('max', totalSteps); - - let progressImageEle = document.querySelector('#progress-image'); - progressImageEle.src = BLANK_IMAGE_URL; - progressImageEle.style.display = showProgressImages ? 'initial': 'none'; -} - -function setProgress(step, totalSteps, src) { - let progressEle = document.querySelector('#progress-bar'); - progressEle.setAttribute('value', step); - - if (src) { - let progressImageEle = document.querySelector('#progress-image'); - progressImageEle.src = src; - } -} - -function resetProgress(hide = true) { - if (hide) { - let progressSectionEle = document.querySelector('#progress-section'); - progressSectionEle.style.display = 'none'; - } - let progressEle = document.querySelector('#progress-bar'); - progressEle.setAttribute('value', 0); -} - function toBase64(file) { return new Promise((resolve, reject) => { const r = new FileReader(); @@ -113,41 +7,17 @@ function toBase64(file) { }); } -function ondragdream(event) { - let dream = event.target.dataset.dream; - event.dataTransfer.setData("dream", dream); -} - -function seedClick(event) { - // Get element - var image = event.target.closest('figure').querySelector('img'); - var dream = JSON.parse(decodeURIComponent(image.dataset.dream)); - - let form = document.querySelector("#generate-form"); - for (const [k, v] of new FormData(form)) { - if (k == 'initimg') { continue; } - let formElem = form.querySelector(`*[name=${k}]`); - formElem.value = dream[k] !== undefined ? dream[k] : formElem.defaultValue; - } - - document.querySelector("#seed").value = dream.seed; - document.querySelector('#iterations').value = 1; // Reset to 1 iteration since we clicked a single image (not a full job) - - // NOTE: leaving this manual for the user for now - it was very confusing with this behavior - // document.querySelector("#with_variations").value = variations || ''; - // if (document.querySelector("#variation_amount").value <= 0) { - // document.querySelector("#variation_amount").value = 0.2; - // } - - saveFields(document.querySelector("#generate-form")); -} - -function appendOutput(src, seed, config, toEnd=false) { +function appendOutput(src, seed, config) { let outputNode = document.createElement("figure"); - let altText = seed.toString() + " | " + config.prompt; + + let variations = config.with_variations; + if (config.variation_amount > 0) { + variations = (variations ? variations + ',' : '') + seed + ':' + config.variation_amount; + } + let baseseed = (config.with_variations || config.variation_amount > 0) ? config.seed : seed; + let altText = baseseed + ' | ' + (variations ? variations + ' | ' : '') + config.prompt; // img needs width and height for lazy loading to work - // TODO: store the full config in a data attribute on the image? const figureContents = ` + height="256"> -
${seed}
+
${seed}
`; outputNode.innerHTML = figureContents; + let figcaption = outputNode.querySelector('figcaption'); - if (toEnd) { - document.querySelector("#results").append(outputNode); - } else { - document.querySelector("#results").prepend(outputNode); - } - document.querySelector("#no-results-message")?.remove(); + // Reload image config + figcaption.addEventListener('click', () => { + let form = document.querySelector("#generate-form"); + for (const [k, v] of new FormData(form)) { + if (k == 'initimg') { continue; } + form.querySelector(`*[name=${k}]`).value = config[k]; + } + + document.querySelector("#seed").value = baseseed; + document.querySelector("#with_variations").value = variations || ''; + if (document.querySelector("#variation_amount").value <= 0) { + document.querySelector("#variation_amount").value = 0.2; + } + + saveFields(document.querySelector("#generate-form")); + }); + + document.querySelector("#results").prepend(outputNode); } function saveFields(form) { @@ -200,109 +79,93 @@ function clearFields(form) { const BLANK_IMAGE_URL = 'data:image/svg+xml,'; async function generateSubmit(form) { + const prompt = document.querySelector("#prompt").value; + // Convert file data to base64 - // TODO: Should probably uplaod files with formdata or something, and store them in the backend? let formData = Object.fromEntries(new FormData(form)); - if (!formData.enable_generate && !formData.enable_init_image) { - gen_label = document.querySelector("label[for=enable_generate]").innerHTML; - initimg_label = document.querySelector("label[for=enable_init_image]").innerHTML; - alert(`Error: one of "${gen_label}" or "${initimg_label}" must be set`); - } - - formData.initimg_name = formData.initimg.name formData.initimg = formData.initimg.name !== '' ? await toBase64(formData.initimg) : null; - // Evaluate all checkboxes - let checkboxes = form.querySelectorAll('input[type=checkbox]'); - checkboxes.forEach(function (checkbox) { - if (checkbox.checked) { - formData[checkbox.name] = 'true'; - } - }); - let strength = formData.strength; let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; - let showProgressImages = formData.progress_images; - // Set enabling flags + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'initial'; + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('max', totalSteps); + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = BLANK_IMAGE_URL; + progressImageEle.style.display = {}.hasOwnProperty.call(formData, 'progress_images') ? 'initial': 'none'; - // Initialize the progress bar - initProgress(totalSteps, showProgressImages); - - // POST, use response to listen for events + // Post as JSON, using Fetch streaming to get results fetch(form.action, { method: form.method, - headers: new Headers({'content-type': 'application/json'}), body: JSON.stringify(formData), - }) - .then(response => response.json()) - .then(data => { - var jobId = data.jobId; - socket.emit('join_room', { 'room': jobId }); + }).then(async (response) => { + const reader = response.body.getReader(); + + let noOutputs = true; + while (true) { + let {value, done} = await reader.read(); + value = new TextDecoder().decode(value); + if (done) { + progressSectionEle.style.display = 'none'; + break; + } + + for (let event of value.split('\n').filter(e => e !== '')) { + const data = JSON.parse(event); + + if (data.event === 'result') { + noOutputs = false; + appendOutput(data.url, data.seed, data.config); + progressEle.setAttribute('value', 0); + progressEle.setAttribute('max', totalSteps); + } else if (data.event === 'upscaling-started') { + document.getElementById("processing_cnt").textContent=data.processed_file_cnt; + document.getElementById("scaling-inprocess-message").style.display = "block"; + } else if (data.event === 'upscaling-done') { + document.getElementById("scaling-inprocess-message").style.display = "none"; + } else if (data.event === 'step') { + progressEle.setAttribute('value', data.step); + if (data.url) { + progressImageEle.src = data.url; + } + } else if (data.event === 'canceled') { + // avoid alerting as if this were an error case + noOutputs = false; + } + } + } + + // Re-enable form, remove no-results-message + form.querySelector('fieldset').removeAttribute('disabled'); + document.querySelector("#prompt").value = prompt; + document.querySelector('progress').setAttribute('value', '0'); + + if (noOutputs) { + alert("Error occurred while generating."); + } }); + // Disable form while generating form.querySelector('fieldset').setAttribute('disabled',''); + document.querySelector("#prompt").value = `Generating: "${prompt}"`; } -function fieldSetEnableChecked(event) { - cb = event.target; - fields = cb.closest('fieldset'); - fields.disabled = !cb.checked; +async function fetchRunLog() { + try { + let response = await fetch('/run_log.json') + const data = await response.json(); + for(let item of data.run_log) { + appendOutput(item.url, item.seed, item); + } + } catch (e) { + console.error(e); + } } -// Socket listeners -socket.on('job_started', (data) => {}) - -socket.on('dream_result', (data) => { - var jobId = data.jobId; - var dreamId = data.dreamId; - var dreamRequest = data.dreamRequest; - var src = 'api/images/' + dreamId; - - priorResultsLoadState.offset += 1; - appendOutput(src, dreamRequest.seed, dreamRequest); - - resetProgress(false); -}) - -socket.on('dream_progress', (data) => { - // TODO: it'd be nice if we could get a seed reported here, but the generator would need to be updated - var step = data.step; - var totalSteps = data.totalSteps; - var jobId = data.jobId; - var dreamId = data.dreamId; - - var progressType = data.progressType - if (progressType === 'GENERATION') { - var src = data.hasProgressImage ? - 'api/intermediates/' + dreamId + '/' + step - : null; - setProgress(step, totalSteps, src); - } else if (progressType === 'UPSCALING_STARTED') { - // step and totalSteps are used for upscale count on this message - document.getElementById("processing_cnt").textContent = step; - document.getElementById("processing_total").textContent = totalSteps; - document.getElementById("scaling-inprocess-message").style.display = "block"; - } else if (progressType == 'UPSCALING_DONE') { - document.getElementById("scaling-inprocess-message").style.display = "none"; - } -}) - -socket.on('job_canceled', (data) => { - resetForm(); - resetProgress(); -}) - -socket.on('job_done', (data) => { - jobId = data.jobId - socket.emit('leave_room', { 'room': jobId }); - - resetForm(); - resetProgress(); -}) - window.onload = async () => { document.querySelector("#prompt").addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.shiftKey) { @@ -320,7 +183,7 @@ window.onload = async () => { saveFields(e.target.form); }); document.querySelector("#reset-seed").addEventListener('click', (e) => { - document.querySelector("#seed").value = 0; + document.querySelector("#seed").value = -1; saveFields(e.target.form); }); document.querySelector("#reset-all").addEventListener('click', (e) => { @@ -332,13 +195,13 @@ window.onload = async () => { loadFields(document.querySelector("#generate-form")); document.querySelector('#cancel-button').addEventListener('click', () => { - fetch('/api/cancel').catch(e => { + fetch('/cancel').catch(e => { console.error(e); }); }); document.documentElement.addEventListener('keydown', (e) => { if (e.key === "Escape") - fetch('/api/cancel').catch(err => { + fetch('/cancel').catch(err => { console.error(err); }); }); @@ -346,51 +209,5 @@ window.onload = async () => { if (!config.gfpgan_model_exists) { document.querySelector("#gfpgan").style.display = 'none'; } - - window.addEventListener("scroll", () => { - if ((window.innerHeight + window.pageYOffset) >= document.body.offsetHeight) { - loadPriorResults(); - } - }); - - - - // Enable/disable forms by checkboxes - document.querySelectorAll("legend > input[type=checkbox]").forEach(function(cb) { - cb.addEventListener('change', fieldSetEnableChecked); - fieldSetEnableChecked({ target: cb}) - }); - - - // Load some of the previous results - loadPriorResults(); - - // Image drop/upload WIP - /* - let drop = document.getElementById('dropper'); - function ondrop(event) { - let dreamData = event.dataTransfer.getData('dream'); - if (dreamData) { - var dream = JSON.parse(decodeURIComponent(dreamData)); - alert(dream.dreamId); - } - }; - - function ondragenter(event) { - event.preventDefault(); - }; - - function ondragover(event) { - event.preventDefault(); - }; - - function ondragleave(event) { - - } - - drop.addEventListener('drop', ondrop); - drop.addEventListener('dragenter', ondragenter); - drop.addEventListener('dragover', ondragover); - drop.addEventListener('dragleave', ondragleave); - */ + await fetchRunLog() }; From ba4892e03fcfab4c8dac71c453cee45f2d4f6568 Mon Sep 17 00:00:00 2001 From: Kevin Schaul Date: Fri, 16 Sep 2022 16:32:18 -0500 Subject: [PATCH 073/238] Zero-pad intermediate image file names (#616) --- ldm/dream/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldm/dream/server.py b/ldm/dream/server.py index cde3957a1f..372d719052 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -228,7 +228,8 @@ class DreamServer(BaseHTTPRequestHandler): nonlocal step_index if opt.progress_images and step % 5 == 0 and step < opt.steps - 1: image = self.model.sample_to_image(sample) - name = f'{prefix}.{opt.seed}.{step_index}.png' + step_index_padded = str(step_index).rjust(len(str(opt.steps)), '0') + name = f'{prefix}.{opt.seed}.{step_index_padded}.png' metadata = f'{opt.prompt} -S{opt.seed} [intermediate]' path = step_writer.save_image_and_prompt_to_png(image, dream_prompt=metadata, name=name) step_index += 1 From 6cab2e0ca025f8541373ab374ab2881ca330ad88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=AF=E4=B8=8D=E6=B8=B8?= <71683364+mefengl@users.noreply.github.com> Date: Sat, 17 Sep 2022 05:32:52 +0800 Subject: [PATCH 074/238] refine env rebuild tip (#611) --- docs/installation/INSTALL_MAC.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 39398c36ac..7ee5653c7a 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -181,7 +181,12 @@ There are several causes of these errors. - Third, if it says you're missing taming you need to rebuild your virtual environment. -`conda env remove -n ldm conda env create -f environment-mac.yaml` +````bash +conda deactivate + +conda env remove -n ldm +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +``` Fourth, If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't From 40b61870f6e18634724cf9b14b1a3492f2ffb680 Mon Sep 17 00:00:00 2001 From: SteveCaruso Date: Fri, 16 Sep 2022 17:42:21 -0400 Subject: [PATCH 075/238] update Intel Mac instructions (#599) Co-authored-by: Lincoln Stein --- docs/installation/INSTALL_MAC.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 7ee5653c7a..71535980f5 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -7,10 +7,7 @@ title: macOS - macOS 12.3 Monterey or later - Python - Patience -- Apple Silicon\* - -\*I haven't tested any of this on Intel Macs but I have read that one person got -it to work, so Apple Silicon might not be requried. +- Apple Silicon or Intel Mac Things have moved really fast and so these instructions change often and are often out-of-date. One of the problems is that there are so many different ways @@ -59,9 +56,13 @@ First get the weights checkpoint download started - it's big: # install python 3, git, cmake, protobuf: brew install cmake protobuf rust -# install miniconda (M1 arm64 version): - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh - /bin/bash Miniconda3-latest-MacOSX-arm64.sh +# install miniconda for M1 arm64: +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh +/bin/bash Miniconda3-latest-MacOSX-arm64.sh + +# OR install miniconda for Intel: +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o Miniconda3-latest-MacOSX-x86_64.sh +/bin/bash Miniconda3-latest-MacOSX-x86_64.sh # EITHER WAY, @@ -82,15 +83,22 @@ brew install cmake protobuf rust ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt -# install packages - PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml - conda activate ldm +# install packages for arm64 +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +conda activate ldm + +# OR install packages for x86_64 +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 conda env create -f environment-mac.yaml +conda activate ldm # only need to do this once python scripts/preload_models.py # run SD! python scripts/dream.py --full_precision # half-precision requires autocast and won't work + +# or run the web interface! +python scripts/dream.py --web ``` The original scripts should work as well. From 37e2418ee0432e64820d3e27cdb81e02a6bf0a94 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Fri, 16 Sep 2022 15:46:57 -0600 Subject: [PATCH 076/238] Added linux to the workflows (#463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added linux to the workflows - rename workflow files Signed-off-by: Ben Alkov * fixes: run on merge to 'main', 'dev'; - reduce dev merge test cases to 1 (1 takes 11 minutes 😯) - fix model cache name Signed-off-by: Ben Alkov * add test prompts to workflows Signed-off-by: Ben Alkov Signed-off-by: Ben Alkov Co-authored-by: James Reynolds Co-authored-by: Ben Alkov Co-authored-by: Lincoln Stein --- .github/workflows/cache-model.yml | 64 ---------------- .github/workflows/create-caches.yml | 70 ++++++++++++++++++ .github/workflows/macos12-miniconda.yml | 80 -------------------- .github/workflows/test-dream-conda.yml | 97 +++++++++++++++++++++++++ tests/dev_prompts.txt | 1 + tests/preflight_prompts.txt | 9 +++ tests/prompts.txt | 1 - 7 files changed, 177 insertions(+), 145 deletions(-) delete mode 100644 .github/workflows/cache-model.yml create mode 100644 .github/workflows/create-caches.yml delete mode 100644 .github/workflows/macos12-miniconda.yml create mode 100644 .github/workflows/test-dream-conda.yml create mode 100644 tests/dev_prompts.txt create mode 100644 tests/preflight_prompts.txt delete mode 100644 tests/prompts.txt diff --git a/.github/workflows/cache-model.yml b/.github/workflows/cache-model.yml deleted file mode 100644 index 2682943eef..0000000000 --- a/.github/workflows/cache-model.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Cache Model -on: - workflow_dispatch -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Create Caches using ${{ matrix.os }} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py \ No newline at end of file diff --git a/.github/workflows/create-caches.yml b/.github/workflows/create-caches.yml new file mode 100644 index 0000000000..951718af1b --- /dev/null +++ b/.github/workflows/create-caches.yml @@ -0,0 +1,70 @@ +name: Create Caches +on: + workflow_dispatch +jobs: + build: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Create Caches on ${{ matrix.os }} conda + runs-on: ${{ matrix.os }} + steps: + - name: Set platform variables + id: vars + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-huggingface-torch + uses: actions/cache@v3 + env: + cache-name: cache-huggingface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-huggingface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py diff --git a/.github/workflows/macos12-miniconda.yml b/.github/workflows/macos12-miniconda.yml deleted file mode 100644 index 18f21277c0..0000000000 --- a/.github/workflows/macos12-miniconda.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Build on ${{ matrix.os }} miniconda - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py - - name: Run the tests - run: | - # Note, can't "activate" via automation, and activation is just env vars and path - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - export PYTORCH_ENABLE_MPS_FALLBACK=1 - $PYTHON_BIN scripts/preload_models.py - mkdir -p outputs/img-samples - time $PYTHON_BIN scripts/dream.py --from_file tests/prompts.txt outputs/img-samples/err.log > outputs/img-samples/out.log - - name: Archive results - uses: actions/upload-artifact@v3 - with: - name: results - path: outputs/img-samples \ No newline at end of file diff --git a/.github/workflows/test-dream-conda.yml b/.github/workflows/test-dream-conda.yml new file mode 100644 index 0000000000..3bd9b24582 --- /dev/null +++ b/.github/workflows/test-dream-conda.yml @@ -0,0 +1,97 @@ +name: Test Dream with Conda +on: + push: + branches: + - 'main' + - 'development' +jobs: + os_matrix: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Test dream.py on ${{ matrix.os }} with conda + runs-on: ${{ matrix.os }} + steps: + - run: | + echo The PR was merged + - name: Set platform variables + id: vars + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-hugginface-torch + uses: actions/cache@v3 + env: + cache-name: cache-hugginface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py +# - name: Run tmate +# uses: mxschmitt/action-tmate@v3 +# timeout-minutes: 30 + - name: Run the tests + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ $(uname) = "Darwin" ]; then + export PYTORCH_ENABLE_MPS_FALLBACK=1 + fi + # Utterly hacky, but I don't know how else to do this + if [[ ${{ github.ref }} == 'refs/heads/master' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/preflight_prompts.txt --full_precision + elif [[ ${{ github.ref }} == 'refs/heads/development' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/dev_prompts.txt --full_precision + fi + mkdir -p outputs/img-samples + - name: Archive results + uses: actions/upload-artifact@v3 + with: + name: results + path: outputs/img-samples diff --git a/tests/dev_prompts.txt b/tests/dev_prompts.txt new file mode 100644 index 0000000000..9ebca4e9f7 --- /dev/null +++ b/tests/dev_prompts.txt @@ -0,0 +1 @@ +banana sushi -Ak_lms -S42 diff --git a/tests/preflight_prompts.txt b/tests/preflight_prompts.txt new file mode 100644 index 0000000000..5c5b8233a1 --- /dev/null +++ b/tests/preflight_prompts.txt @@ -0,0 +1,9 @@ +banana sushi -Ak_lms -S42 +banana sushi -Addim -S42 +banana sushi -Ak_lms -W640 -H480 -S42 +banana sushi -Ak_lms -S42 -G1 -U 2 0.5 +banana sushi -Ak_lms -S42 -v0.2 -n3 +banana sushi -Ak_lms -S42 -V1349749425:0.1,4145759947:0.1 +snake -I outputs/preflight/000006.4145759947.png -S42 +snake -I outputs/preflight/000006.4145759947.png -S42 -W640 -H640 --fit +strawberry sushi -I./image-and-mask.png -S42 -f0.9 -s100 -C15 \ No newline at end of file diff --git a/tests/prompts.txt b/tests/prompts.txt deleted file mode 100644 index 955220a5e6..0000000000 --- a/tests/prompts.txt +++ /dev/null @@ -1 +0,0 @@ -test trending on artstation -s 1 -S 1 From 622db491b255d2ab4e15d930b20782c06b04c14b Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 18:25:43 -0400 Subject: [PATCH 077/238] change tensor length to 768 per #572 --- ldm/modules/embedding_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ldm/modules/embedding_manager.py b/ldm/modules/embedding_manager.py index b579bcd885..09e6f495ab 100644 --- a/ldm/modules/embedding_manager.py +++ b/ldm/modules/embedding_manager.py @@ -82,7 +82,9 @@ class EmbeddingManager(nn.Module): get_embedding_for_clip_token, embedder.transformer.text_model.embeddings, ) - token_dim = 1280 + # per bug report #572 + #token_dim = 1280 + token_dim = 768 else: # using LDM's BERT encoder self.is_clip = False get_token_for_string = partial( From fb7a9f37e4f1208e4156457b0f401ff9a182328e Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" <3804335+santisbon@users.noreply.github.com> Date: Fri, 16 Sep 2022 18:35:50 -0500 Subject: [PATCH 078/238] Set temporary instructions Set temporary instructions to use the branch that can currently be containerized. --- docker-build/Dockerfile | 65 +++++++--------- docker-build/entrypoint.sh | 8 +- docs/installation/INSTALL_DOCKER.md | 112 ++++++++++++++-------------- 3 files changed, 90 insertions(+), 95 deletions(-) diff --git a/docker-build/Dockerfile b/docker-build/Dockerfile index 05dcb822cc..8922cdfde5 100644 --- a/docker-build/Dockerfile +++ b/docker-build/Dockerfile @@ -3,66 +3,57 @@ FROM debian ARG gsd ENV GITHUB_STABLE_DIFFUSION $gsd -ARG sdreq="requirements-linux-arm64.txt" -ENV SD_REQ $sdreq +ARG rsd +ENV REQS $rsd -ARG condaarch -ENV ARCH $condaarch +ARG cs +ENV CONDA_SUBDIR $cs -WORKDIR / +ENV PIP_EXISTS_ACTION="w" # TODO: Optimize image size -COPY entrypoint.sh anaconda.sh . SHELL ["/bin/bash", "-c"] -# Update and apt 446 MB +WORKDIR / RUN apt update && apt upgrade -y \ && apt install -y \ git \ + libgl1-mesa-glx \ + libglib2.0-0 \ pip \ python3 \ - wget + wget \ + && git clone $GITHUB_STABLE_DIFFUSION -# install Anaconda or Miniconda 610 MB +# Install Anaconda or Miniconda +COPY anaconda.sh . RUN bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash -# SD repo 105 MB -RUN git clone $GITHUB_STABLE_DIFFUSION - +# SD WORKDIR /stable-diffusion +RUN source ~/.bashrc \ + && conda create -y --name ldm && conda activate ldm \ + && conda config --env --set subdir $CONDA_SUBDIR \ + && pip3 install -r $REQS \ + && pip3 install basicsr facexlib realesrgan \ + && mkdir models/ldm/stable-diffusion-v1 \ + && ln -s "/data/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt -# SD env 2.3 GB !!! -RUN PIP_EXISTS_ACTION="w" \ - && CONDA_SUBDIR=$ARCH \ - && source ~/.bashrc && conda create -y --name ldm && conda activate ldm \ - && conda config --env --set subdir $ARCH \ - && pip3 install -r $SD_REQ \ - && mkdir models/ldm/stable-diffusion-v1 - -# Face restoration prerequisites 200 MB -RUN apt install -y libgl1-mesa-glx libglib2.0-0 - -WORKDIR / - -# Face restoreation repo 12 MB +# Face restoreation # by default expected in a sibling directory to stable-diffusion +WORKDIR / RUN git clone https://github.com/TencentARC/GFPGAN.git WORKDIR /GFPGAN - -# Face restoration env 608 MB -RUN pip3 install basicsr facexlib \ - && pip3 install -r requirements.txt \ +RUN pip3 install -r requirements.txt \ && python3 setup.py develop \ - # to enhance the background (non-face) regions and do upscaling - && pip3 install realesrgan \ - # pre-trained model needed for face restoration - && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + # && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + && ln -s "/data/GFPGANv1.3.pth" experiments/pretrained_models/GFPGANv1.3.pth WORKDIR /stable-diffusion - -# Preload models 2 GB RUN python3 scripts/preload_models.py -ENTRYPOINT ["/entrypoint.sh"] +WORKDIR / +COPY entrypoint.sh . +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/docker-build/entrypoint.sh b/docker-build/entrypoint.sh index c7cc9af8d3..f47e6669e0 100755 --- a/docker-build/entrypoint.sh +++ b/docker-build/entrypoint.sh @@ -1,10 +1,10 @@ #!/bin/bash -ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt + cd /stable-diffusion -conda activate ldm if [ $# -eq 0 ]; then - python3 scripts/dream.py --full_precision -o /data + python3 scripts/dream.py --full_precision -o /data + # bash else python3 scripts/dream.py --full_precision -o /data "$@" -fi +fi \ No newline at end of file diff --git a/docs/installation/INSTALL_DOCKER.md b/docs/installation/INSTALL_DOCKER.md index 9a124b4e52..34974e0f2f 100644 --- a/docs/installation/INSTALL_DOCKER.md +++ b/docs/installation/INSTALL_DOCKER.md @@ -1,95 +1,101 @@ +# Before you begin -Table of Contents -================= +- For end users: Install Stable Diffusion locally using the instructions for your OS. +- For developers: For container-related development tasks or for enabling easy deployment to other environments (on-premises or cloud), follow these instructions. For general use, install locally to leverage your machine's GPU. -* [Step 1 - Get the Model](#step-1---get-the-model) -* [Step 2 - Installation](#step-2---installation) - * [On a Linux container](#on-a-linux-container) - * [Why containers?](#why-containers) - * [Prerequisites](#prerequisites) - * [Setup](#setup) -* [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) - * [Startup](#startup) - * [Text to Image](#text-to-image) - * [Image to Image](#image-to-image) - * [Web Interface](#web-interface) - * [Notes](#notes) +# Why containers? -# Step 1 - Get the Model -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. -You'll need to create an account but it's quick and free. +They provide a flexible, reliable way to build and deploy Stable Diffusion. You'll also use a Docker volume to store the largest model files and image outputs as a first step in decoupling storage and compute. Future enhancements can do this for other assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important. -# Step 2 - Installation +You can specify the target platform when building the image and running the container. You'll also need to specify the Stable Diffusion requirements file that matches the container's OS and the architecture it will run on. -## On a Linux container +Developers on Apple silicon (M1/M2): You [can't access your GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to another environment with NVIDIA GPUs on-premises or in the cloud. -### Why containers? -They provide a flexible, reliable way to build and deploy Stable Diffusion. We also use a Docker volume to store the largest model file and image outputs as a first step in decoupling storage and compute. Future enhancements will do this for other model files and assets. See [Processes](https://12factor.net/processes) under the Twelve-Factor App methodology for details on why running applications in such a stateless fashion is important. +# Installation on a Linux container -This example uses a Mac M1/M2 (arm64) but you can specify the platform and architecture as parameters when building the image and running the container. You'll also need to specify the Stable Diffusion requirements file that matches your OS and architecture e.g. Linux on an arm64 chip if running a Linux container on Apple silicon. +## Prerequisites -The steps would be the same on an amd64 machine with NVIDIA GPUs as for an arm64 Mac; the platform is configurable. You [can't access the Mac M1/M2 GPU cores from Docker containers](https://github.com/pytorch/pytorch/issues/81224) and performance is reduced compared with running it directly on macOS but for development purposes it's fine. Once you're done with development tasks on your laptop you can build for the target platform and architecture and deploy to an environment with NVIDIA GPUs on-premises or in the cloud. +### Get the data files -### Prerequisites -[Install Docker](https://github.com/santisbon/guides#docker) +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download the model file ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. + +Also download the face restoration model. +```Shell +cd ~/Downloads +wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth +``` + +### Install [Docker](https://github.com/santisbon/guides#docker) On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. -Create a Docker volume for the downloaded model file +## Setup + +Set the fork you want to use and other variables. +```Shell +TAG_STABLE_DIFFUSION="santisbon/stable-diffusion" +PLATFORM="linux/arm64" +GITHUB_STABLE_DIFFUSION="-b orig-gfpgan https://github.com/santisbon/stable-diffusion.git" +REQS_STABLE_DIFFUSION="requirements-linux-arm64.txt" +CONDA_SUBDIR="osx-arm64" + +echo $TAG_STABLE_DIFFUSION +echo $PLATFORM +echo $GITHUB_STABLE_DIFFUSION +echo $REQS_STABLE_DIFFUSION +echo $CONDA_SUBDIR ``` + +Create a Docker volume for the downloaded model files. +```Shell docker volume create my-vol ``` -Copy the model file (we'll need it at run time) to the Docker volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +Copy the data files to the Docker volume using a lightweight Linux container. We'll need the models at run time. You just need to create the container with the mountpoint; no need to run this dummy container. ```Shell -docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine +cd ~/Downloads # or wherever you saved the files + +docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine -cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data +docker cp GFPGANv1.3.pth dummy:/data ``` -### Setup -Set the fork you want to use. -Download the Miniconda installer (we'll need it at build time). Replace the URL with the version matching your system. +Get the repo and download the Miniconda installer (we'll need it at build time). Replace the URL with the version matching your container OS and the architecture it will run on. ```Shell -GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" - cd ~ git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion/docker-build chmod +x entrypoint.sh - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh ``` Build the Docker image. Give it any tag ```-t``` that you want. -Tip: Check that your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. -```condaarch``` will restrict the conda environment to the right architecture when installing packages. It can take on: ```linux-64```, ```osx-64```, ```osx-arm64```. +Choose the Linux container's host platform: x86-64/Intel is ```amd64```. Apple silicon is ```arm64```. If deploying the container to the cloud to leverage powerful GPU instances you'll be on amd64 hardware but if you're just trying this out locally on Apple silicon choose arm64. +The application uses libraries that need to match the host environment so use the appropriate requirements file. +Tip: Check that your shell session has the env variables set above. ```Shell -docker build -t santisbon/stable-diffusion \ ---platform linux/arm64 \ ---build-arg condaarch="osx-arm64" \ +docker build -t $TAG_STABLE_DIFFUSION \ +--platform $PLATFORM \ --build-arg gsd=$GITHUB_STABLE_DIFFUSION \ ---build-arg sdreq="requirements-linux-arm64.txt" \ +--build-arg rsd=$REQS_STABLE_DIFFUSION \ +--build-arg cs=$CONDA_SUBDIR \ . ``` -Run a container using your built image e.g. +Run a container using your built image. +Tip: Make sure you've created and populated the Docker volume (above). ```Shell docker run -it \ --rm \ ---platform linux/arm64 \ +--platform $PLATFORM \ --name stable-diffusion \ --hostname stable-diffusion \ --mount source=my-vol,target=/data \ ---expose 9090 \ ---publish 9090:9090 \ -santisbon/stable-diffusion +$TAG_STABLE_DIFFUSION ``` -Tip: Make sure you've created the Docker volume (above) - -# Step 3 - Usage (time to have fun) +# Usage (time to have fun) ## Startup If you're on a **Linux container** the ```dream``` script is **automatically started** and the output dir set to the Docker volume you created earlier. @@ -115,12 +121,11 @@ The prompt can be in quotes or not. ```Shell dream> The hulk fighting with sheldon cooper -s5 -n1 dream> "woman closeup highly detailed" -s 150 -# Reuse previous seed and apply face restoration (if you installed GFPGAN) -dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 +# Reuse previous seed and apply face restoration +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 ``` You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. -The ```-U``` option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). If you're on a container the output is set to the Docker volume. You can copy it wherever you want. You can download it from the Docker Desktop app, Volumes, my-vol, data. @@ -175,5 +180,4 @@ The original scripts should work as well. python3 scripts/orig_scripts/txt2img.py --help python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1 --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray" python3 scripts/orig_scripts/txt2img.py --ddim_steps 5 --n_iter 1 --n_samples 1 --plms --prompt "ocean" # or --klms -``` - +``` \ No newline at end of file From df95a7ddf2e8eb055552617cec258d0cdc371360 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 19:58:16 -0400 Subject: [PATCH 079/238] respect --outdir again; fix issue #628 --- ldm/dream/args.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/ldm/dream/args.py b/ldm/dream/args.py index 9201f46dfa..24056d340e 100644 --- a/ldm/dream/args.py +++ b/ldm/dream/args.py @@ -216,9 +216,7 @@ class Args(object): # the arg value. For example, the --grid and --individual options are a little # funny because of their push/pull relationship. This is how to handle it. if name=='grid': - return value_arg or value_cmd # arg supersedes cmd - if name=='individual': - return value_cmd or value_arg # cmd supersedes arg + return not cmd_switches.individual and value_arg # arg supersedes cmd if value_cmd is not None: return value_cmd else: @@ -294,11 +292,6 @@ class Args(object): action='store_true', help='Place images in subdirectories named after the prompt.', ) - render_group.add_argument( - '--seamless', - action='store_true', - help='Change the model to seamless tiling (circular) mode', - ) render_group.add_argument( '--grid', '-g', @@ -416,8 +409,8 @@ class Args(object): help='generate a grid' ) render_group.add_argument( - '--individual', '-i', + '--individual', action='store_true', help='override command-line --grid setting and generate individual images' ) @@ -448,7 +441,6 @@ class Args(object): '--outdir', '-o', type=str, - default='outputs/img-samples', help='Directory to save generated images and a log of prompts and seeds', ) img2img_group.add_argument( From df4c80f177b892b4190b932fc30aa5ec65649608 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 16 Sep 2022 19:58:45 -0400 Subject: [PATCH 080/238] respect --outdir again; fix issue #628 --- .github/workflows/cache-model.yml | 64 ---------------- .github/workflows/create-caches.yml | 70 ++++++++++++++++++ .github/workflows/macos12-miniconda.yml | 80 -------------------- .github/workflows/test-dream-conda.yml | 97 +++++++++++++++++++++++++ docs/installation/INSTALL_MAC.md | 35 ++++++--- ldm/dream/server.py | 3 +- ldm/modules/embedding_manager.py | 4 +- tests/dev_prompts.txt | 1 + tests/preflight_prompts.txt | 9 +++ tests/prompts.txt | 1 - 10 files changed, 206 insertions(+), 158 deletions(-) delete mode 100644 .github/workflows/cache-model.yml create mode 100644 .github/workflows/create-caches.yml delete mode 100644 .github/workflows/macos12-miniconda.yml create mode 100644 .github/workflows/test-dream-conda.yml create mode 100644 tests/dev_prompts.txt create mode 100644 tests/preflight_prompts.txt delete mode 100644 tests/prompts.txt diff --git a/.github/workflows/cache-model.yml b/.github/workflows/cache-model.yml deleted file mode 100644 index 2682943eef..0000000000 --- a/.github/workflows/cache-model.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Cache Model -on: - workflow_dispatch -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Create Caches using ${{ matrix.os }} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py \ No newline at end of file diff --git a/.github/workflows/create-caches.yml b/.github/workflows/create-caches.yml new file mode 100644 index 0000000000..951718af1b --- /dev/null +++ b/.github/workflows/create-caches.yml @@ -0,0 +1,70 @@ +name: Create Caches +on: + workflow_dispatch +jobs: + build: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Create Caches on ${{ matrix.os }} conda + runs-on: ${{ matrix.os }} + steps: + - name: Set platform variables + id: vars + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-huggingface-torch + uses: actions/cache@v3 + env: + cache-name: cache-huggingface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-huggingface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py diff --git a/.github/workflows/macos12-miniconda.yml b/.github/workflows/macos12-miniconda.yml deleted file mode 100644 index 18f21277c0..0000000000 --- a/.github/workflows/macos12-miniconda.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Build on ${{ matrix.os }} miniconda - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py - - name: Run the tests - run: | - # Note, can't "activate" via automation, and activation is just env vars and path - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - export PYTORCH_ENABLE_MPS_FALLBACK=1 - $PYTHON_BIN scripts/preload_models.py - mkdir -p outputs/img-samples - time $PYTHON_BIN scripts/dream.py --from_file tests/prompts.txt outputs/img-samples/err.log > outputs/img-samples/out.log - - name: Archive results - uses: actions/upload-artifact@v3 - with: - name: results - path: outputs/img-samples \ No newline at end of file diff --git a/.github/workflows/test-dream-conda.yml b/.github/workflows/test-dream-conda.yml new file mode 100644 index 0000000000..3bd9b24582 --- /dev/null +++ b/.github/workflows/test-dream-conda.yml @@ -0,0 +1,97 @@ +name: Test Dream with Conda +on: + push: + branches: + - 'main' + - 'development' +jobs: + os_matrix: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Test dream.py on ${{ matrix.os }} with conda + runs-on: ${{ matrix.os }} + steps: + - run: | + echo The PR was merged + - name: Set platform variables + id: vars + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-hugginface-torch + uses: actions/cache@v3 + env: + cache-name: cache-hugginface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py +# - name: Run tmate +# uses: mxschmitt/action-tmate@v3 +# timeout-minutes: 30 + - name: Run the tests + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ $(uname) = "Darwin" ]; then + export PYTORCH_ENABLE_MPS_FALLBACK=1 + fi + # Utterly hacky, but I don't know how else to do this + if [[ ${{ github.ref }} == 'refs/heads/master' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/preflight_prompts.txt --full_precision + elif [[ ${{ github.ref }} == 'refs/heads/development' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/dev_prompts.txt --full_precision + fi + mkdir -p outputs/img-samples + - name: Archive results + uses: actions/upload-artifact@v3 + with: + name: results + path: outputs/img-samples diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 39398c36ac..71535980f5 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -7,10 +7,7 @@ title: macOS - macOS 12.3 Monterey or later - Python - Patience -- Apple Silicon\* - -\*I haven't tested any of this on Intel Macs but I have read that one person got -it to work, so Apple Silicon might not be requried. +- Apple Silicon or Intel Mac Things have moved really fast and so these instructions change often and are often out-of-date. One of the problems is that there are so many different ways @@ -59,9 +56,13 @@ First get the weights checkpoint download started - it's big: # install python 3, git, cmake, protobuf: brew install cmake protobuf rust -# install miniconda (M1 arm64 version): - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh - /bin/bash Miniconda3-latest-MacOSX-arm64.sh +# install miniconda for M1 arm64: +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh +/bin/bash Miniconda3-latest-MacOSX-arm64.sh + +# OR install miniconda for Intel: +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o Miniconda3-latest-MacOSX-x86_64.sh +/bin/bash Miniconda3-latest-MacOSX-x86_64.sh # EITHER WAY, @@ -82,15 +83,22 @@ brew install cmake protobuf rust ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt -# install packages - PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml - conda activate ldm +# install packages for arm64 +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +conda activate ldm + +# OR install packages for x86_64 +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 conda env create -f environment-mac.yaml +conda activate ldm # only need to do this once python scripts/preload_models.py # run SD! python scripts/dream.py --full_precision # half-precision requires autocast and won't work + +# or run the web interface! +python scripts/dream.py --web ``` The original scripts should work as well. @@ -181,7 +189,12 @@ There are several causes of these errors. - Third, if it says you're missing taming you need to rebuild your virtual environment. -`conda env remove -n ldm conda env create -f environment-mac.yaml` +````bash +conda deactivate + +conda env remove -n ldm +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +``` Fourth, If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't diff --git a/ldm/dream/server.py b/ldm/dream/server.py index cde3957a1f..372d719052 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -228,7 +228,8 @@ class DreamServer(BaseHTTPRequestHandler): nonlocal step_index if opt.progress_images and step % 5 == 0 and step < opt.steps - 1: image = self.model.sample_to_image(sample) - name = f'{prefix}.{opt.seed}.{step_index}.png' + step_index_padded = str(step_index).rjust(len(str(opt.steps)), '0') + name = f'{prefix}.{opt.seed}.{step_index_padded}.png' metadata = f'{opt.prompt} -S{opt.seed} [intermediate]' path = step_writer.save_image_and_prompt_to_png(image, dream_prompt=metadata, name=name) step_index += 1 diff --git a/ldm/modules/embedding_manager.py b/ldm/modules/embedding_manager.py index b579bcd885..09e6f495ab 100644 --- a/ldm/modules/embedding_manager.py +++ b/ldm/modules/embedding_manager.py @@ -82,7 +82,9 @@ class EmbeddingManager(nn.Module): get_embedding_for_clip_token, embedder.transformer.text_model.embeddings, ) - token_dim = 1280 + # per bug report #572 + #token_dim = 1280 + token_dim = 768 else: # using LDM's BERT encoder self.is_clip = False get_token_for_string = partial( diff --git a/tests/dev_prompts.txt b/tests/dev_prompts.txt new file mode 100644 index 0000000000..9ebca4e9f7 --- /dev/null +++ b/tests/dev_prompts.txt @@ -0,0 +1 @@ +banana sushi -Ak_lms -S42 diff --git a/tests/preflight_prompts.txt b/tests/preflight_prompts.txt new file mode 100644 index 0000000000..5c5b8233a1 --- /dev/null +++ b/tests/preflight_prompts.txt @@ -0,0 +1,9 @@ +banana sushi -Ak_lms -S42 +banana sushi -Addim -S42 +banana sushi -Ak_lms -W640 -H480 -S42 +banana sushi -Ak_lms -S42 -G1 -U 2 0.5 +banana sushi -Ak_lms -S42 -v0.2 -n3 +banana sushi -Ak_lms -S42 -V1349749425:0.1,4145759947:0.1 +snake -I outputs/preflight/000006.4145759947.png -S42 +snake -I outputs/preflight/000006.4145759947.png -S42 -W640 -H640 --fit +strawberry sushi -I./image-and-mask.png -S42 -f0.9 -s100 -C15 \ No newline at end of file diff --git a/tests/prompts.txt b/tests/prompts.txt deleted file mode 100644 index 955220a5e6..0000000000 --- a/tests/prompts.txt +++ /dev/null @@ -1 +0,0 @@ -test trending on artstation -s 1 -S 1 From d8d30ab4cbde8d8694c884e3c97d28638313fe48 Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 15:28:37 +0200 Subject: [PATCH 081/238] separate toc by disabling toc.integrate this way the leftside menu does not look so bloated and users can find what they are looking for much faster --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index c7c50834d8..f0f64db502 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,8 +13,8 @@ copyright: Copyright © 2022 Lincoln D. Stein # Configuration theme: name: material - features: - - toc.integrate + # features: + # - toc.integrate palette: - media: '(prefers-color-scheme: light)' primary: blue From fa7fe382b7beb5f50f53e3d39c7f97ddf85b4cb3 Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 13:45:42 +0200 Subject: [PATCH 082/238] remove protobuf from brew install docs --- docs/installation/INSTALL_MAC.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 71535980f5..67c2c025b2 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -53,8 +53,8 @@ First get the weights checkpoint download started - it's big: # OR, # 2. Installing standalone -# install python 3, git, cmake, protobuf: -brew install cmake protobuf rust +# install python 3, git, cmake: +brew install cmake rust # install miniconda for M1 arm64: curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh From fefcdffb55e1784efdaeb3e26436e4f7fcfb5ea0 Mon Sep 17 00:00:00 2001 From: tildebyte <337875+tildebyte@users.noreply.github.com> Date: Fri, 16 Sep 2022 20:21:19 -0400 Subject: [PATCH 083/238] fix(readme): switch last-commit badge to last DEV commit (#626) - switch badge service to badgen, as I couldn't figure out shields.io Signed-off-by: Ben Alkov Signed-off-by: Ben Alkov --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2dacbc7866..8f88a5c9d2 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,16 @@

- last-commit - stars -
- issues - pull-requests + release + stars + forks +
+ CI status on main + CI status on dev + last-dev-commit +
+ open-issues + open-prs

This is a fork of [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion), the open From 443fcd030f437c3a98c0e93e456ccbefe0a7593e Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 20:34:34 +0200 Subject: [PATCH 084/238] change printWidth for markdown files to 80 since 80 is the defautl lenght --- .prettierrc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.prettierrc.yaml b/.prettierrc.yaml index 68eae1ba15..ce4b99a07b 100644 --- a/.prettierrc.yaml +++ b/.prettierrc.yaml @@ -5,9 +5,9 @@ singleQuote: true quoteProps: as-needed embeddedLanguageFormatting: auto overrides: - - files: "*.md" + - files: '*.md' options: proseWrap: always - printWidth: 100 + printWidth: 80 parser: markdown cursorOffset: -1 From cd494c2f6caa52aa38be1bc2275b2889d17a9de2 Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 14:07:17 +0200 Subject: [PATCH 085/238] fix some markdown violations as well --- docs/installation/INSTALL_MAC.md | 54 +++++++++++++++++++------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 67c2c025b2..f87aa6f99e 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -45,11 +45,11 @@ First get the weights checkpoint download started - it's big: # NOW EITHER DO # 1. Installing alongside pyenv - brew install pyenv-virtualenv # you might have this from before, no problem - pyenv install anaconda3-2022.05 - pyenv virtualenv anaconda3-2022.05 - eval "$(pyenv init -)" - pyenv activate anaconda3-2022.05 +brew install pyenv-virtualenv # you might have this from before, no problem +pyenv install anaconda3-2022.05 +pyenv virtualenv anaconda3-2022.05 +eval "$(pyenv init -)" +pyenv activate anaconda3-2022.05 # OR, # 2. Installing standalone @@ -69,19 +69,17 @@ curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o M # continue from here # clone the repo - git clone https://github.com/lstein/stable-diffusion.git - cd stable-diffusion +git clone https://github.com/lstein/stable-diffusion.git +cd stable-diffusion -# # wait until the checkpoint file has downloaded, then proceed -# # create symlink to checkpoint - mkdir -p models/ldm/stable-diffusion-v1/ +mkdir -p models/ldm/stable-diffusion-v1/ - PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt +PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt - ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt +ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt # install packages for arm64 PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml @@ -189,7 +187,7 @@ There are several causes of these errors. - Third, if it says you're missing taming you need to rebuild your virtual environment. -````bash +```bash conda deactivate conda env remove -n ldm @@ -291,8 +289,10 @@ list all of the `python` / `python3` things found in `$PATH` instead of just the one that will be executed by default. To do that, add the `-a` switch to `which`: - % which -a python3 - ... +```bash +% which -a python3 +... +``` ### Debugging? @@ -300,17 +300,21 @@ Tired of waiting for your renders to finish before you can see if it works? Reduce the steps! The image quality will be horrible but at least you'll get quick feedback. - python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 +```bash +python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 +``` ### OSError: Can't load tokenizer for 'openai/clip-vit-large-patch14'... - python scripts/preload_models.py +```bash +python scripts/preload_models.py +``` ### "The operator [name] is not current implemented for the MPS device." (sic) Example error. -``` +```bash ... NotImplementedError: The operator 'aten::_index_put_impl_' is not current implemented for the MPS device. If you want this op to be added in priority @@ -330,7 +334,9 @@ The lstein branch includes this fix in I have not seen this error because I had Rust installed on my computer before I started playing with Stable Diffusion. The fix is to install Rust. - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` ### How come `--seed` doesn't work? @@ -347,8 +353,9 @@ still working on it. ### libiomp5.dylib error? - OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized. - +```bash +OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized. +``` You are likely using an Intel package by mistake. Be sure to run conda with the environment variable `CONDA_SUBDIR=osx-arm64`, like so: @@ -428,7 +435,10 @@ This is a 32-bit vs 16-bit problem. ### The processor must support the Intel bla bla bla What? Intel? On an Apple Silicon? -`bash Intel MKL FATAL ERROR: This system does not meet the minimum requirements for use of the Intel(R) Math Kernel Library. The processor must support the Intel(R) Supplemental Streaming SIMD Extensions 3 (Intel(R) SSSE3) instructions. The processor must support the Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) instructions. The processor must support the Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions. ` + +```bash +Intel MKL FATAL ERROR: This system does not meet the minimum requirements for use of the Intel(R) Math Kernel Library. The processor must support the Intel(R) Supplemental Streaming SIMD Extensions 3 (Intel(R) SSSE3) instructions. The processor must support the Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) instructions. The processor must support the Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions. +``` This is due to the Intel `mkl` package getting picked up when you try to install something that depends on it-- Rosetta can translate some Intel instructions but From 3c732500e717b2aefdd919a04597d524347240fd Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 21:30:21 +0200 Subject: [PATCH 086/238] many updatedes to INSTALL_MAC.md --- docs/installation/INSTALL_MAC.md | 186 +++++++++++++++++++------------ 1 file changed, 115 insertions(+), 71 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index f87aa6f99e..4f8a5477f6 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -9,18 +9,21 @@ title: macOS - Patience - Apple Silicon or Intel Mac -Things have moved really fast and so these instructions change often and are -often out-of-date. One of the problems is that there are so many different ways -to run this. +Things have moved really fast and so these instructions change often which makes +them outdated pretty fast. One of the problems is that there are so many +different ways to run this. We are trying to build a testing setup so that when we make changes it doesn't always break. -How to (this hasn't been 100% tested yet): +## How to -First get the weights checkpoint download started - it's big: +(this hasn't been 100% tested yet) -1. Sign up at https://huggingface.co +First get the weights checkpoint download started since it's big and will take +some time: + +1. Sign up at [huggingface.co](https://huggingface.co) 2. Go to the [Stable diffusion diffusion model page](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) 3. Accept the terms and click Access Repository: @@ -28,24 +31,26 @@ First get the weights checkpoint download started - it's big: [sd-v1-4.ckpt (4.27 GB)](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/blob/main/sd-v1-4.ckpt) and note where you have saved it (probably the Downloads folder) - While that is downloading, open Terminal and run the following commands one - at a time. +While that is downloading, open Terminal and run the following commands one at a +time. -```bash +```{ .bash .annotate } # install brew (and Xcode command line tools): +/bin/bash -c \ + "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - -# Now there are two different routes to get the Python (miniconda) environment up and running: +# Now there are two different routes to get the Python (miniconda) environment +# up and running: +# # 1. Alongside pyenv # 2. No pyenv # # If you don't know what we are talking about, choose 2. -# + # NOW EITHER DO # 1. Installing alongside pyenv -brew install pyenv-virtualenv # you might have this from before, no problem +brew install pyenv-virtualenv # (1)! pyenv install anaconda3-2022.05 pyenv virtualenv anaconda3-2022.05 eval "$(pyenv init -)" @@ -53,7 +58,7 @@ pyenv activate anaconda3-2022.05 # OR, # 2. Installing standalone -# install python 3, git, cmake: +# install cmake and rust: brew install cmake rust # install miniconda for M1 arm64: @@ -77,16 +82,21 @@ cd stable-diffusion # create symlink to checkpoint mkdir -p models/ldm/stable-diffusion-v1/ -PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt +PATH_TO_CKPT="$HOME/Downloads" # (2)! -ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt +ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" \ + models/ldm/stable-diffusion-v1/model.ckpt # install packages for arm64 -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \ + conda env create \ + -f environment-mac.yaml conda activate ldm # OR install packages for x86_64 -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 conda env create -f environment-mac.yaml +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 \ + conda env create \ + -f environment-mac.yaml conda activate ldm # only need to do this once @@ -99,25 +109,32 @@ python scripts/dream.py --full_precision # half-precision requires autocast and python scripts/dream.py --web ``` +1. you might have this from before, no problem +2. or wherever you saved sd-v1-4.ckpt +3. half-precision requires autocast and won't work + The original scripts should work as well. ```bash -python scripts/orig_scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms +python scripts/orig_scripts/txt2img.py \ + --prompt "a photograph of an astronaut riding a horse" \ + --plms ``` -Note, +Note: ```bash export PIP_EXISTS_ACTION=w ``` -is a precaution to fix +is a precaution to fix a problem where ```bash -conda env create -f environment-mac.yaml +conda env create \ + -f environment-mac.yaml ``` -never finishing in some situations. So it isn't required but wont hurt. +did never finish in some situations. So it isn't required but wont hurt. After you follow all the instructions and run dream.py you might get several errors. Here's the errors I've seen and found solutions for. @@ -130,10 +147,10 @@ Be sure to specify 1 sample and 1 iteration. ```bash python ./scripts/orig_scripts/txt2img.py \ - --prompt "ocean" \ - --ddim_steps 5 \ - --n_samples 1 \ - --n_iter 1 + --prompt "ocean" \ + --ddim_steps 5 \ + --n_samples 1 \ + --n_iter 1 ``` --- @@ -151,55 +168,76 @@ solution please One debugging step is to update to the latest version of PyTorch nightly. ```bash -conda install pytorch torchvision torchaudio -c pytorch-nightly +conda install \ + pytorch \ + torchvision \ + torchaudio \ + -c pytorch-nightly ``` If it takes forever to run ```bash -conda env create -f environment-mac.yaml +conda env create \ + -f environment-mac.yaml ``` -you could try to run `git clean -f` followed by: +you could try to run: -`conda clean --yes --all` +```bash +git clean -f +conda clean \ + --yes \ + --all +``` Or you could try to completley reset Anaconda: ```bash -conda update --force-reinstall -y -n base -c defaults conda +conda update \ + --force-reinstall \ + -y \ + -n base \ + -c defaults conda ``` --- ### "No module named cv2", torch, 'ldm', 'transformers', 'taming', etc -There are several causes of these errors. +There are several causes of these errors: -- First, did you remember to `conda activate ldm`? If your terminal prompt - begins with "(ldm)" then you activated it. If it begins with "(base)" or - something else you haven't. +1. Did you remember to `conda activate ldm`? If your terminal prompt begins with + "(ldm)" then you activated it. If it begins with "(base)" or something else + you haven't. -- Second, you might've run `./scripts/preload_models.py` or `./scripts/dream.py` - instead of `python ./scripts/preload_models.py` or - `python ./scripts/dream.py`. The cause of this error is long so it's below. +2. You might've run `./scripts/preload_models.py` or `./scripts/dream.py` + instead of `python ./scripts/preload_models.py` or + `python ./scripts/dream.py`. The cause of this error is long so it's below. -- Third, if it says you're missing taming you need to rebuild your virtual - environment. + -```bash -conda deactivate +3. if it says you're missing taming you need to rebuild your virtual + environment. -conda env remove -n ldm -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml -``` + ```bash + conda deactivate -Fourth, If you have activated the ldm virtual environment and tried rebuilding -it, maybe the problem could be that I have something installed that you don't -and you'll just need to manually install it. Make sure you activate the virtual -environment so it installs there instead of globally. + conda env remove -n ldm + PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \ + conda env create \ + -f environment-mac.yaml + ``` -`conda activate ldm pip install _name_` +4. If you have activated the ldm virtual environment and tried rebuilding it, + maybe the problem could be that I have something installed that you don't and + you'll just need to manually install it. Make sure you activate the virtual + environment so it installs there instead of globally. + + ```bash + conda activate ldm + pip install + ``` You might also need to install Rust (I mention this again below). @@ -259,21 +297,22 @@ output of `python3 -V` and `python -V`. /Users/name/miniforge3/envs/ldm/bin/python ``` -The above is what you'll see if you have miniforge and you've correctly -activated the ldm environment, and you used option 2 in the setup instructions -above ("no pyenv"). +The above is what you'll see if you have miniforge and correctly activated the +ldm environment, while usingd option 2 in the setup instructions above ("no +pyenv"). + +If you otherwise used the first option ("alongside pyenv"), you will get this +prompt: ```bash (anaconda3-2022.05) % which python /Users/name/.pyenv/shims/python ``` -... and the above is what you'll see if you used option 1 ("Alongside pyenv"). - It's all a mess and you should know [how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) -if you want to fix it. Here's a brief hint of all the ways you can modify it -(don't really have the time to explain it all here). +if you want to fix it. Here's a brief hint of the most common ways you can +modify it (don't really have the time to explain it all here). - ~/.zshrc - ~/.bash_profile @@ -281,19 +320,20 @@ if you want to fix it. Here's a brief hint of all the ways you can modify it - /etc/paths.d - /etc/path -Which one you use will depend on what you have installed except putting a file -in /etc/paths.d is what I prefer to do. +Which one you use will depend on what you have installed, except putting a file +in /etc/paths.d - which also is the way I prefer to do. Finally, to answer the question posed by this section's title, it may help to list all of the `python` / `python3` things found in `$PATH` instead of just the -one that will be executed by default. To do that, add the `-a` switch to -`which`: +first hit. To do so, add the `-a` switch to `which`: ```bash % which -a python3 ... ``` +This will show a list of all binaries which are actually available in your PATH. + ### Debugging? Tired of waiting for your renders to finish before you can see if it works? @@ -301,10 +341,14 @@ Reduce the steps! The image quality will be horrible but at least you'll get quick feedback. ```bash -python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 +python ./scripts/txt2img.py \ + --prompt "ocean" \ + --ddim_steps 5 \ + --n_samples 1 \ + --n_iter 1 ``` -### OSError: Can't load tokenizer for 'openai/clip-vit-large-patch14'... +### OSError: Can't load tokenizer for 'openai/clip-vit-large-patch14' ```bash python scripts/preload_models.py @@ -312,10 +356,7 @@ python scripts/preload_models.py ### "The operator [name] is not current implemented for the MPS device." (sic) -Example error. - -```bash - +```bash title="example error" ... NotImplementedError: The operator 'aten::_index_put_impl_' is not current implemented for the MPS device. If you want this op to be added in priority during the prototype phase of this feature, please comment on @@ -323,7 +364,6 @@ during the prototype phase of this feature, please comment on As a temporary fix, you can set the environment variable `PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. WARNING: this will be slower than running natively on MPS. - ``` The lstein branch includes this fix in @@ -335,7 +375,10 @@ I have not seen this error because I had Rust installed on my computer before I started playing with Stable Diffusion. The fix is to install Rust. ```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +curl \ + --proto '=https' \ + --tlsv1.2 \ + -sSf https://sh.rustup.rs | sh ``` ### How come `--seed` doesn't work? @@ -356,6 +399,7 @@ still working on it. ```bash OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized. ``` + You are likely using an Intel package by mistake. Be sure to run conda with the environment variable `CONDA_SUBDIR=osx-arm64`, like so: From ae963fcfdc24760220e4c1ec5ec8f03b2017d398 Mon Sep 17 00:00:00 2001 From: mauwii Date: Fri, 16 Sep 2022 22:26:42 +0200 Subject: [PATCH 087/238] fix codeblock anotations --- docs/installation/INSTALL_MAC.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 4f8a5477f6..5c3bc42082 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -50,7 +50,7 @@ time. # NOW EITHER DO # 1. Installing alongside pyenv -brew install pyenv-virtualenv # (1)! +brew install pyenv-virtualenv # (1) pyenv install anaconda3-2022.05 pyenv virtualenv anaconda3-2022.05 eval "$(pyenv init -)" @@ -82,7 +82,7 @@ cd stable-diffusion # create symlink to checkpoint mkdir -p models/ldm/stable-diffusion-v1/ -PATH_TO_CKPT="$HOME/Downloads" # (2)! +PATH_TO_CKPT="$HOME/Downloads" # (2) ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" \ models/ldm/stable-diffusion-v1/model.ckpt @@ -103,7 +103,7 @@ conda activate ldm python scripts/preload_models.py # run SD! -python scripts/dream.py --full_precision # half-precision requires autocast and won't work +python scripts/dream.py --full_precision # (3) # or run the web interface! python scripts/dream.py --web From 17e755e0628174105cc79f8c8fceb9b42f050ef3 Mon Sep 17 00:00:00 2001 From: mauwii Date: Sat, 17 Sep 2022 03:05:13 +0200 Subject: [PATCH 088/238] shorten long commands in codeblocks this way it looks much better in MkDocs Codeblocks (and ofc still works) also remove a markdown styled link from a bash-codeblock --- docs/installation/INSTALL_MAC.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 5c3bc42082..5e1968f038 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -62,11 +62,13 @@ pyenv activate anaconda3-2022.05 brew install cmake rust # install miniconda for M1 arm64: -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh \ + -o Miniconda3-latest-MacOSX-arm64.sh /bin/bash Miniconda3-latest-MacOSX-arm64.sh # OR install miniconda for Intel: -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o Miniconda3-latest-MacOSX-x86_64.sh +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh \ + -o Miniconda3-latest-MacOSX-x86_64.sh /bin/bash Miniconda3-latest-MacOSX-x86_64.sh @@ -222,7 +224,6 @@ There are several causes of these errors: ```bash conda deactivate - conda env remove -n ldm PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \ conda env create \ @@ -360,7 +361,7 @@ python scripts/preload_models.py ... NotImplementedError: The operator 'aten::_index_put_impl_' is not current implemented for the MPS device. If you want this op to be added in priority during the prototype phase of this feature, please comment on -[https://github.com/pytorch/pytorch/issues/77764](https://github.com/pytorch/pytorch/issues/77764). +https://github.com/pytorch/pytorch/issues/77764. As a temporary fix, you can set the environment variable `PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. WARNING: this will be slower than running natively on MPS. From 103b30f915208f1501109905dc0f9ca2744e5a13 Mon Sep 17 00:00:00 2001 From: mauwii Date: Sat, 17 Sep 2022 03:49:11 +0200 Subject: [PATCH 089/238] update index.md with the new badges --- docs/index.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/index.md b/docs/index.md index bdde3cabd7..0a909c5d83 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,26 +3,30 @@ title: Home --- -

Stable Diffusion Dream Script

- +

- last-commit - stars -
- issues - pull-requests + release + stars + forks +
+ CI status on main + CI status on dev + last-dev-commit +
+ open-issues + open-prs

This is a fork of [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion), the open From 42072fc15c6c70ac33ad7d34d834e55307b27dc1 Mon Sep 17 00:00:00 2001 From: blessedcoolant <54517381+blessedcoolant@users.noreply.github.com> Date: Sat, 17 Sep 2022 14:12:35 +1200 Subject: [PATCH 090/238] Bug Fixes --- ldm/dream/args.py | 15 ++++++++++++--- scripts/dream.py | 10 +++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ldm/dream/args.py b/ldm/dream/args.py index 24056d340e..04cea509a7 100644 --- a/ldm/dream/args.py +++ b/ldm/dream/args.py @@ -105,6 +105,7 @@ class Args(object): try: elements = shlex.split(command) except ValueError: + import sys, traceback print(traceback.format_exc(), file=sys.stderr) return switches = [''] @@ -266,6 +267,17 @@ class Args(object): default='stable-diffusion-1.4', help='Indicates which diffusion model to load. (currently "stable-diffusion-1.4" (default) or "laion400m")', ) + model_group.add_argument( + '--sampler', + '-A', + '-m', + dest='sampler_name', + type=str, + choices=SAMPLER_CHOICES, + metavar='SAMPLER_NAME', + help=f'Switch to a different sampler. Supported samplers: {", ".join(SAMPLER_CHOICES)}', + default='k_lms', + ) model_group.add_argument( '-F', '--full_precision', @@ -386,14 +398,12 @@ class Args(object): '--width', type=int, help='Image width, multiple of 64', - default=512 ) render_group.add_argument( '-H', '--height', type=int, help='Image height, multiple of 64', - default=512, ) render_group.add_argument( '-C', @@ -429,7 +439,6 @@ class Args(object): choices=SAMPLER_CHOICES, metavar='SAMPLER_NAME', help=f'Switch to a different sampler. Supported samplers: {", ".join(SAMPLER_CHOICES)}', - default='k_lms', ) render_group.add_argument( '-t', diff --git a/scripts/dream.py b/scripts/dream.py index 35de70d650..b8a5e5a84b 100644 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -123,7 +123,7 @@ def main_loop(gen, opt, infile): if command.startswith(('#', '//')): continue - if command.startswith('q '): + if len(command.strip()) == 1 and command.startswith('q'): done = True break @@ -138,7 +138,7 @@ def main_loop(gen, opt, infile): parser.print_help() continue if len(opt.prompt) == 0: - print('Try again with a prompt!') + print('\nTry again with a prompt!') continue # retrieve previous value! @@ -191,14 +191,14 @@ def main_loop(gen, opt, infile): if not os.path.exists(opt.outdir): os.makedirs(opt.outdir) current_outdir = opt.outdir - elif prompt_as_dir: + elif opt.prompt_as_dir: # sanitize the prompt to a valid folder name subdir = path_filter.sub('_', opt.prompt)[:name_max].rstrip(' .') # truncate path to maximum allowed length # 27 is the length of '######.##########.##.png', plus two separators and a NUL subdir = subdir[:(path_max - 27 - len(os.path.abspath(opt.outdir)))] - current_outdir = os.path.join(outdir, subdir) + current_outdir = os.path.join(opt.outdir, subdir) print('Writing files to directory: "' + current_outdir + '"') @@ -206,7 +206,7 @@ def main_loop(gen, opt, infile): if not os.path.exists(current_outdir): os.makedirs(current_outdir) else: - current_outdir = outdir + current_outdir = opt.outdir # Here is where the images are actually generated! last_results = [] From 7b28b5c9a1aed56fdba8247ae9f5457b24bd60ef Mon Sep 17 00:00:00 2001 From: mauwii Date: Sat, 17 Sep 2022 05:17:21 +0200 Subject: [PATCH 091/238] update format in EMBIGGEN.md to make use of mkdocs --- docs/features/EMBIGGEN.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/features/EMBIGGEN.md b/docs/features/EMBIGGEN.md index 70f35fe758..ae12f29b6d 100644 --- a/docs/features/EMBIGGEN.md +++ b/docs/features/EMBIGGEN.md @@ -1,4 +1,8 @@ -# **Embiggen -- upscale your images on limited memory machines** +--- +title: Embiggen +--- + +**upscale your images on limited memory machines** GFPGAN and Real-ESRGAN are both memory intensive. In order to avoid crashes and memory overloads during the Stable Diffusion process, @@ -16,7 +20,7 @@ face restore a particular generated image, pass it again with the same prompt and generated seed along with the `-U` and `-G` prompt arguments to perform those actions. -## Embiggen +## Embiggen If you wanted to be able to do more (pixels) without running out of VRAM, or you want to upscale with details that couldn't possibly appear @@ -37,7 +41,7 @@ it's similar to that, except it can work up to an arbitrarily large size has extra logic to re-run any number of the tile sub-sections of the image if for example a small part of a huge run got messed up. -**Usage** +## Usage `-embiggen ` @@ -95,12 +99,12 @@ Tiles are numbered starting with one, and left-to-right, top-to-bottom. So, if you are generating a 3x3 tiled image, the middle row would be `4 5 6`. -**Example Usage** +## Example Usage Running Embiggen with 512x512 tiles on an existing image, scaling up by a factor of 2.5x; and doing the same again (default ESRGAN strength is 0.75, default overlap between tiles is 0.25): -``` +```bash dream > a photo of a forest at sunset -s 100 -W 512 -H 512 -I outputs/forest.png -f 0.4 -embiggen 2.5 dream > a photo of a forest at sunset -s 100 -W 512 -H 512 -I outputs/forest.png -f 0.4 -embiggen 2.5 0.75 0.25 ``` @@ -112,11 +116,11 @@ If there weren't enough clouds in the sky of that forest you just made 512x512 tiles with 0.25 overlaps wide) we can replace that top row of tiles: -``` +```bash dream> a photo of puffy clouds over a forest at sunset -s 100 -W 512 -H 512 -I outputs/000002.seed.png -f 0.5 -embiggen_tiles 1 2 3 ``` -**Note** +## Note Because the same prompt is used on all the tiled images, and the model doesn't have the context of anything outside the tile being run - it @@ -131,4 +135,4 @@ are more abstract. Because this is (relatively) fast, you can also always create a few Embiggen'ed images and manually composite them to preserve the best parts from each. -Author: [Travco](https://github.com/travco) \ No newline at end of file +Author: [Travco](https://github.com/travco) From b9183b00a03f51654f01892f944a4d15d1040dbb Mon Sep 17 00:00:00 2001 From: mauwii Date: Sat, 17 Sep 2022 06:34:38 +0200 Subject: [PATCH 092/238] add Content tabs --- docs/installation/INSTALL_MAC.md | 123 ++++++++++++++++++------------- 1 file changed, 70 insertions(+), 53 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 5e1968f038..2ab7c6de83 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -31,50 +31,57 @@ some time: [sd-v1-4.ckpt (4.27 GB)](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/blob/main/sd-v1-4.ckpt) and note where you have saved it (probably the Downloads folder) -While that is downloading, open Terminal and run the following commands one at a -time. +While that is downloading, open a Terminal and run the following commands: -```{ .bash .annotate } -# install brew (and Xcode command line tools): +```bash title="install brew (and Xcode command line tools)" /bin/bash -c \ "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` -# Now there are two different routes to get the Python (miniconda) environment -# up and running: -# -# 1. Alongside pyenv -# 2. No pyenv -# -# If you don't know what we are talking about, choose 2. +!!! quote "Conda Installation" + Now there are two different ways to set up the Python (miniconda) environment: -# NOW EITHER DO -# 1. Installing alongside pyenv + 1. Standalone + 2. with pyenv + + If you don't know what we are talking about, choose Standalone -brew install pyenv-virtualenv # (1) -pyenv install anaconda3-2022.05 -pyenv virtualenv anaconda3-2022.05 -eval "$(pyenv init -)" -pyenv activate anaconda3-2022.05 + === "Standalone" -# OR, -# 2. Installing standalone -# install cmake and rust: -brew install cmake rust + ```bash + # install cmake and rust: + brew install cmake rust + ``` -# install miniconda for M1 arm64: -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh \ - -o Miniconda3-latest-MacOSX-arm64.sh -/bin/bash Miniconda3-latest-MacOSX-arm64.sh + === "M1 arm64" -# OR install miniconda for Intel: -curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh \ - -o Miniconda3-latest-MacOSX-x86_64.sh -/bin/bash Miniconda3-latest-MacOSX-x86_64.sh + ```bash + # install miniconda for M1 arm64: + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh \ + -o Miniconda3-latest-MacOSX-arm64.sh + /bin/bash Miniconda3-latest-MacOSX-arm64.sh + ``` + === "Intel x86_64" -# EITHER WAY, -# continue from here + ```bash + # OR install miniconda for Intel: + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh \ + -o Miniconda3-latest-MacOSX-x86_64.sh + /bin/bash Miniconda3-latest-MacOSX-x86_64.sh + ``` + === "with pyenv" + + ```bash + brew install pyenv-virtualenv # you might already have this installed, no problem + pyenv install anaconda3-2022.05 + pyenv virtualenv anaconda3-2022.05 + eval "$(pyenv init -)" + pyenv activate anaconda3-2022.05 + ``` + +```{.bash .annotate} # clone the repo git clone https://github.com/lstein/stable-diffusion.git cd stable-diffusion @@ -84,45 +91,55 @@ cd stable-diffusion # create symlink to checkpoint mkdir -p models/ldm/stable-diffusion-v1/ -PATH_TO_CKPT="$HOME/Downloads" # (2) +PATH_TO_CKPT="$HOME/Downloads" # (1) ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" \ models/ldm/stable-diffusion-v1/model.ckpt -# install packages for arm64 -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \ - conda env create \ - -f environment-mac.yaml -conda activate ldm +``` -# OR install packages for x86_64 -PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 \ - conda env create \ - -f environment-mac.yaml -conda activate ldm +1. or wherever you saved sd-v1-4.ckpt +!!! quote "Please install the propper package for your Architecture:" + + === "M1 arm64" + + ```bash + PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 \ + conda env create \ + -f environment-mac.yaml \ + && conda activate ldm + ``` + + === "Intel x86_64" + + ```bash + PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-x86_64 \ + conda env create \ + -f environment-mac.yaml \ + && conda activate ldm + ``` + +```{.bash .annotate} # only need to do this once python scripts/preload_models.py -# run SD! -python scripts/dream.py --full_precision # (3) +# now you can run SD in CLI mode +python scripts/dream.py --full_precision # (1) # or run the web interface! python scripts/dream.py --web -``` -1. you might have this from before, no problem -2. or wherever you saved sd-v1-4.ckpt -3. half-precision requires autocast and won't work - -The original scripts should work as well. - -```bash +# The original scripts should work as well. python scripts/orig_scripts/txt2img.py \ --prompt "a photograph of an astronaut riding a horse" \ --plms ``` +1. half-precision requires autocast which is unfortunatelly incompatible + +--- + Note: ```bash From b89aadb3c957c2b87b58334a16734f0fbba2f293 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 17 Sep 2022 00:57:35 -0400 Subject: [PATCH 093/238] fix crash on second prompt #636 --- ldm/dream/args.py | 21 +++++++++++++-------- ldm/dream/pngwriter.py | 1 - scripts/dream.py | 5 +---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ldm/dream/args.py b/ldm/dream/args.py index 04cea509a7..e9b70a7199 100644 --- a/ldm/dream/args.py +++ b/ldm/dream/args.py @@ -190,10 +190,10 @@ class Args(object): pass if cmd_switches and arg_switches and name=='__dict__': - a = arg_switches.__dict__ - a.update(cmd_switches.__dict__) - return a - + return self._merge_dict( + arg_switches.__dict__, + cmd_switches.__dict__, + ) try: return object.__getattribute__(self,name) except AttributeError: @@ -218,10 +218,7 @@ class Args(object): # funny because of their push/pull relationship. This is how to handle it. if name=='grid': return not cmd_switches.individual and value_arg # arg supersedes cmd - if value_cmd is not None: - return value_cmd - else: - return value_arg + return value_cmd if value_cmd is not None else value_arg def __setattr__(self,name,value): if name.startswith('_'): @@ -229,6 +226,14 @@ class Args(object): else: self._cmd_switches.__dict__[name] = value + def _merge_dict(self,dict1,dict2): + new_dict = {} + for k in set(list(dict1.keys())+list(dict2.keys())): + value1 = dict1.get(k,None) + value2 = dict2.get(k,None) + new_dict[k] = value2 if value2 is not None else value1 + return new_dict + def _create_arg_parser(self): ''' This defines all the arguments used on the command line when you launch diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index 9a2a8bc816..5cda259357 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -34,7 +34,6 @@ class PngWriter: # saves image named _image_ to outdir/name, writing metadata from prompt # returns full path of output def save_image_and_prompt_to_png(self, image, dream_prompt, name, metadata=None): - print(f'self.outdir={self.outdir}, name={name}') path = os.path.join(self.outdir, name) info = PngImagePlugin.PngInfo() info.add_text('Dream', dream_prompt) diff --git a/scripts/dream.py b/scripts/dream.py index b8a5e5a84b..ad10551bbc 100644 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -132,10 +132,7 @@ def main_loop(gen, opt, infile): ): # in case a stored prompt still contains the !dream command command.replace('!dream','',1) - try: - parser = opt.parse_cmd(command) - except SystemExit: - parser.print_help() + if opt.parse_cmd(command) is None: continue if len(opt.prompt) == 0: print('\nTry again with a prompt!') From 5b692f47205780cfc2019992c78ce31b509f4578 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 17 Sep 2022 01:14:00 -0400 Subject: [PATCH 094/238] include width and height in png dream prompt --- scripts/dream.py | 7 +++++++ 1 file changed, 7 insertions(+) mode change 100644 => 100755 scripts/dream.py diff --git a/scripts/dream.py b/scripts/dream.py old mode 100644 new mode 100755 index ad10551bbc..f147008d78 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -100,6 +100,7 @@ def main_loop(gen, opt, infile): done = False path_filter = re.compile(r'[<>:"/\\|?*]') last_results = list() + model_config = OmegaConf.load(opt.conf)[opt.model] # os.pathconf is not available on Windows if hasattr(os, 'pathconf'): @@ -138,6 +139,12 @@ def main_loop(gen, opt, infile): print('\nTry again with a prompt!') continue + # width and height are set by model if not specified + if not opt.width: + opt.width = model_config.width + if not opt.height: + opt.height = model_config.height + # retrieve previous value! if opt.init_img is not None and re.match('^-\\d+$', opt.init_img): try: From 31daf1f0d794cac1eba526d2e4c17fc8ea1f4867 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 17 Sep 2022 01:32:31 -0400 Subject: [PATCH 095/238] preload_models.py now downloads gfpgan model --- scripts/preload_models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/preload_models.py b/scripts/preload_models.py index ba7b242ece..794a552134 100755 --- a/scripts/preload_models.py +++ b/scripts/preload_models.py @@ -10,6 +10,7 @@ import sys import transformers import os import warnings +import urllib.request transformers.logging.set_verbosity_error() @@ -81,6 +82,16 @@ if gfpgan: print('...success') except Exception: import traceback + print('Error loading ESRGAN:') + print(traceback.format_exc()) + try: + import urllib.request + model_path = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth' + model_dest = 'src/gfpgan/experiments/pretrained_models/GFPGANv1.3.pth' + print('downloading gfpgan model file...') + urllib.request.urlretrieve(model_path,model_dest) + except Exception: + import traceback print('Error loading GFPGAN:') print(traceback.format_exc()) From ed8ee8c690a69e2521d137c9ad599c9be5d3d911 Mon Sep 17 00:00:00 2001 From: mauwii Date: Sat, 17 Sep 2022 07:55:55 +0200 Subject: [PATCH 096/238] update install-mode refference --- docs/installation/INSTALL_MAC.md | 42 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/installation/INSTALL_MAC.md b/docs/installation/INSTALL_MAC.md index 2ab7c6de83..c45512e77b 100644 --- a/docs/installation/INSTALL_MAC.md +++ b/docs/installation/INSTALL_MAC.md @@ -100,7 +100,7 @@ ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" \ 1. or wherever you saved sd-v1-4.ckpt -!!! quote "Please install the propper package for your Architecture:" +!!! quote "Select the propper Architecture" === "M1 arm64" @@ -138,15 +138,9 @@ python scripts/orig_scripts/txt2img.py \ 1. half-precision requires autocast which is unfortunatelly incompatible ---- - Note: -```bash -export PIP_EXISTS_ACTION=w -``` - -is a precaution to fix a problem where +`#!bash export PIP_EXISTS_ACTION=w` is a precaution to fix a problem where ```bash conda env create \ @@ -155,11 +149,13 @@ conda env create \ did never finish in some situations. So it isn't required but wont hurt. -After you follow all the instructions and run dream.py you might get several -errors. Here's the errors I've seen and found solutions for. - --- +## Common problems + +After you followed all the instructions and try to run dream.py, you might +get several errors. Here's the errors I've seen and found solutions for. + ### Is it slow? Be sure to specify 1 sample and 1 iteration. @@ -316,11 +312,9 @@ output of `python3 -V` and `python -V`. ``` The above is what you'll see if you have miniforge and correctly activated the -ldm environment, while usingd option 2 in the setup instructions above ("no -pyenv"). +ldm environment, while usingd the standalone setup instructions above. -If you otherwise used the first option ("alongside pyenv"), you will get this -prompt: +If you otherwise installed via pyenv, you will get this result: ```bash (anaconda3-2022.05) % which python @@ -352,6 +346,8 @@ first hit. To do so, add the `-a` switch to `which`: This will show a list of all binaries which are actually available in your PATH. +--- + ### Debugging? Tired of waiting for your renders to finish before you can see if it works? @@ -366,12 +362,16 @@ python ./scripts/txt2img.py \ --n_iter 1 ``` +--- + ### OSError: Can't load tokenizer for 'openai/clip-vit-large-patch14' ```bash python scripts/preload_models.py ``` +--- + ### "The operator [name] is not current implemented for the MPS device." (sic) ```bash title="example error" @@ -387,6 +387,8 @@ WARNING: this will be slower than running natively on MPS. The lstein branch includes this fix in [environment-mac.yaml](https://github.com/lstein/stable-diffusion/blob/main/environment-mac.yaml). +--- + ### "Could not build wheels for tokenizers" I have not seen this error because I had Rust installed on my computer before I @@ -399,6 +401,8 @@ curl \ -sSf https://sh.rustup.rs | sh ``` +--- + ### How come `--seed` doesn't work? First this: @@ -432,6 +436,8 @@ is a metapackage designed to prevent this, by making it impossible to install Do _not_ use `os.environ['KMP_DUPLICATE_LIB_OK']='True'` or equivalents as this masks the underlying issue of using Intel packages. +--- + ### Not enough memory This seems to be a common problem and is probably the underlying problem for a @@ -443,6 +449,8 @@ how that would affect the quality of the images though. See [this issue](https://github.com/CompVis/stable-diffusion/issues/71). +--- + ### "Error: product of dimension sizes > 2\*\*31'" This error happens with img2img, which I haven't played with too much yet. But I @@ -457,6 +465,8 @@ BTW, 2\*\*31-1 = is also 32-bit signed [LONG_MAX](https://en.wikipedia.org/wiki/C_data_types) in C. +--- + ### I just got Rickrolled! Do I have a virus? You don't have a virus. It's part of the project. Here's @@ -469,6 +479,8 @@ call this "computer vision", sheesh). Actually, this could be happening because there's not enough RAM. You could try the `model.half()` suggestion or specify smaller output images. +--- + ### My images come out black We might have this fixed, we are still testing. From 89540f293bfab3295ede7a0b73fa277f2a463a36 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 17 Sep 2022 02:01:55 -0400 Subject: [PATCH 097/238] Restored static files needed for new flask/react web server WARNING: old web server will no longer display correct interface. --- static/dream_web/index.css | 61 ++++-- static/dream_web/index.html | 274 +++++++++++++++---------- static/dream_web/index.js | 399 ++++++++++++++++++++++++++---------- 3 files changed, 497 insertions(+), 237 deletions(-) diff --git a/static/dream_web/index.css b/static/dream_web/index.css index 51f0f267c3..25a0994a3d 100644 --- a/static/dream_web/index.css +++ b/static/dream_web/index.css @@ -1,3 +1,8 @@ +:root { + --fields-dark:#DCDCDC; + --fields-light:#F5F5F5; +} + * { font-family: 'Arial'; font-size: 100%; @@ -18,15 +23,26 @@ fieldset { border: none; line-height: 2.2em; } +fieldset > legend { + width: auto; + margin-left: 0; + margin-right: auto; + font-weight:bold; +} select, input { margin-right: 10px; padding: 2px; } +input:disabled { + cursor:auto; +} input[type=submit] { + cursor: pointer; background-color: #666; color: white; } input[type=checkbox] { + cursor: pointer; margin-right: 0px; width: 20px; height: 20px; @@ -87,11 +103,11 @@ header h1 { } #results img { border-radius: 5px; - object-fit: cover; + object-fit: contain; + background-color: var(--fields-dark); } #fieldset-config { line-height:2em; - background-color: #F0F0F0; } input[type="number"] { width: 60px; @@ -118,35 +134,46 @@ label { #progress-image { width: 30vh; height: 30vh; + object-fit: contain; + background-color: var(--fields-dark); } #cancel-button { cursor: pointer; color: red; } -#basic-parameters { - background-color: #EEEEEE; -} #txt2img { - background-color: #DCDCDC; + background-color: var(--fields-dark); } #variations { - background-color: #EEEEEE; + background-color: var(--fields-light); +} +#initimg { + background-color: var(--fields-dark); } #img2img { - background-color: #DCDCDC; + background-color: var(--fields-light); } -#gfpgan { - background-color: #EEEEEE; +#initimg > :not(legend) { + background-color: var(--fields-light); + margin: .5em; +} + +#postprocess, #initimg { + display:flex; + flex-wrap:wrap; + padding: 0; + margin-top: 1em; + background-color: var(--fields-dark); +} +#postprocess > fieldset, #initimg > * { + flex-grow: 1; +} +#postprocess > fieldset { + background-color: var(--fields-dark); } #progress-section { - background-color: #F5F5F5; -} -.section-header { - text-align: left; - font-weight: bold; - padding: 0 0 0 0; + background-color: var(--fields-light); } #no-results-message:not(:only-child) { display: none; } - diff --git a/static/dream_web/index.html b/static/dream_web/index.html index 1e194c0205..9dbd213669 100644 --- a/static/dream_web/index.html +++ b/static/dream_web/index.html @@ -1,102 +1,152 @@ - - Stable Diffusion Dream Server - - - - - - - - -
-

Stable Diffusion Dream Server

-
- For news and support for this web service, visit our GitHub site -
-
-
-
-
- -
-
-
Basic options
- - - - - - - - - - -
- - - - - - - - - - - - - - - - -
-
-
Image-to-image options
+ + Stable Diffusion Dream Server + + + + + + + + + + + +
+

Stable Diffusion Dream Server

+
+ For news and support for this web service, visit our GitHub + site +
+
+ +
+ + +
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+ + + + +
+
+
+ + + + +
-
- - - - -
+
+
+ + + + + + + + +
+ +
-
Post-processing options
- - - + + + + + + +
+
+ + + + +
- -
-
-
- - -
- -
- Postprocessing...1/3 -
- -
- -
-
-

No results...

+
+ + +
+
+
+ + +
+ +
+ Postprocessing...1/3
- - +
+ +
+
+ + + diff --git a/static/dream_web/index.js b/static/dream_web/index.js index ac68034920..5de690297d 100644 --- a/static/dream_web/index.js +++ b/static/dream_web/index.js @@ -1,3 +1,109 @@ +const socket = io(); + +var priorResultsLoadState = { + page: 0, + pages: 1, + per_page: 10, + total: 20, + offset: 0, // number of items generated since last load + loading: false, + initialized: false +}; + +function loadPriorResults() { + // Fix next page by offset + let offsetPages = priorResultsLoadState.offset / priorResultsLoadState.per_page; + priorResultsLoadState.page += offsetPages; + priorResultsLoadState.pages += offsetPages; + priorResultsLoadState.total += priorResultsLoadState.offset; + priorResultsLoadState.offset = 0; + + if (priorResultsLoadState.loading) { + return; + } + + if (priorResultsLoadState.page >= priorResultsLoadState.pages) { + return; // Nothing more to load + } + + // Load + priorResultsLoadState.loading = true + let url = new URL('/api/images', document.baseURI); + url.searchParams.append('page', priorResultsLoadState.initialized ? priorResultsLoadState.page + 1 : priorResultsLoadState.page); + url.searchParams.append('per_page', priorResultsLoadState.per_page); + fetch(url.href, { + method: 'GET', + headers: new Headers({'content-type': 'application/json'}) + }) + .then(response => response.json()) + .then(data => { + priorResultsLoadState.page = data.page; + priorResultsLoadState.pages = data.pages; + priorResultsLoadState.per_page = data.per_page; + priorResultsLoadState.total = data.total; + + data.items.forEach(function(dreamId, index) { + let src = 'api/images/' + dreamId; + fetch('/api/images/' + dreamId + '/metadata', { + method: 'GET', + headers: new Headers({'content-type': 'application/json'}) + }) + .then(response => response.json()) + .then(metadata => { + let seed = metadata.seed || 0; // TODO: Parse old metadata + appendOutput(src, seed, metadata, true); + }); + }); + + // Load until page is full + if (!priorResultsLoadState.initialized) { + if (document.body.scrollHeight <= window.innerHeight) { + loadPriorResults(); + } + } + }) + .finally(() => { + priorResultsLoadState.loading = false; + priorResultsLoadState.initialized = true; + }); +} + +function resetForm() { + var form = document.getElementById('generate-form'); + form.querySelector('fieldset').removeAttribute('disabled'); +} + +function initProgress(totalSteps, showProgressImages) { + // TODO: Progress could theoretically come from multiple jobs at the same time (in the future) + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'initial'; + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('max', totalSteps); + + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = BLANK_IMAGE_URL; + progressImageEle.style.display = showProgressImages ? 'initial': 'none'; +} + +function setProgress(step, totalSteps, src) { + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('value', step); + + if (src) { + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = src; + } +} + +function resetProgress(hide = true) { + if (hide) { + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'none'; + } + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('value', 0); +} + function toBase64(file) { return new Promise((resolve, reject) => { const r = new FileReader(); @@ -7,17 +113,41 @@ function toBase64(file) { }); } -function appendOutput(src, seed, config) { - let outputNode = document.createElement("figure"); - - let variations = config.with_variations; - if (config.variation_amount > 0) { - variations = (variations ? variations + ',' : '') + seed + ':' + config.variation_amount; +function ondragdream(event) { + let dream = event.target.dataset.dream; + event.dataTransfer.setData("dream", dream); +} + +function seedClick(event) { + // Get element + var image = event.target.closest('figure').querySelector('img'); + var dream = JSON.parse(decodeURIComponent(image.dataset.dream)); + + let form = document.querySelector("#generate-form"); + for (const [k, v] of new FormData(form)) { + if (k == 'initimg') { continue; } + let formElem = form.querySelector(`*[name=${k}]`); + formElem.value = dream[k] !== undefined ? dream[k] : formElem.defaultValue; } - let baseseed = (config.with_variations || config.variation_amount > 0) ? config.seed : seed; - let altText = baseseed + ' | ' + (variations ? variations + ' | ' : '') + config.prompt; + + document.querySelector("#seed").value = dream.seed; + document.querySelector('#iterations').value = 1; // Reset to 1 iteration since we clicked a single image (not a full job) + + // NOTE: leaving this manual for the user for now - it was very confusing with this behavior + // document.querySelector("#with_variations").value = variations || ''; + // if (document.querySelector("#variation_amount").value <= 0) { + // document.querySelector("#variation_amount").value = 0.2; + // } + + saveFields(document.querySelector("#generate-form")); +} + +function appendOutput(src, seed, config, toEnd=false) { + let outputNode = document.createElement("figure"); + let altText = seed.toString() + " | " + config.prompt; // img needs width and height for lazy loading to work + // TODO: store the full config in a data attribute on the image? const figureContents = ` + height="256" + draggable="true" + ondragstart="ondragdream(event, this)" + data-dream="${encodeURIComponent(JSON.stringify(config))}" + data-dreamId="${encodeURIComponent(config.dreamId)}"> -
${seed}
+
${seed}
`; outputNode.innerHTML = figureContents; - let figcaption = outputNode.querySelector('figcaption'); - // Reload image config - figcaption.addEventListener('click', () => { - let form = document.querySelector("#generate-form"); - for (const [k, v] of new FormData(form)) { - if (k == 'initimg') { continue; } - form.querySelector(`*[name=${k}]`).value = config[k]; - } - - document.querySelector("#seed").value = baseseed; - document.querySelector("#with_variations").value = variations || ''; - if (document.querySelector("#variation_amount").value <= 0) { - document.querySelector("#variation_amount").value = 0.2; - } - - saveFields(document.querySelector("#generate-form")); - }); - - document.querySelector("#results").prepend(outputNode); + if (toEnd) { + document.querySelector("#results").append(outputNode); + } else { + document.querySelector("#results").prepend(outputNode); + } + document.querySelector("#no-results-message")?.remove(); } function saveFields(form) { @@ -79,93 +200,109 @@ function clearFields(form) { const BLANK_IMAGE_URL = 'data:image/svg+xml,'; async function generateSubmit(form) { - const prompt = document.querySelector("#prompt").value; - // Convert file data to base64 + // TODO: Should probably uplaod files with formdata or something, and store them in the backend? let formData = Object.fromEntries(new FormData(form)); + if (!formData.enable_generate && !formData.enable_init_image) { + gen_label = document.querySelector("label[for=enable_generate]").innerHTML; + initimg_label = document.querySelector("label[for=enable_init_image]").innerHTML; + alert(`Error: one of "${gen_label}" or "${initimg_label}" must be set`); + } + + formData.initimg_name = formData.initimg.name formData.initimg = formData.initimg.name !== '' ? await toBase64(formData.initimg) : null; - let strength = formData.strength; - let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; - - let progressSectionEle = document.querySelector('#progress-section'); - progressSectionEle.style.display = 'initial'; - let progressEle = document.querySelector('#progress-bar'); - progressEle.setAttribute('max', totalSteps); - let progressImageEle = document.querySelector('#progress-image'); - progressImageEle.src = BLANK_IMAGE_URL; - - progressImageEle.style.display = {}.hasOwnProperty.call(formData, 'progress_images') ? 'initial': 'none'; - - // Post as JSON, using Fetch streaming to get results - fetch(form.action, { - method: form.method, - body: JSON.stringify(formData), - }).then(async (response) => { - const reader = response.body.getReader(); - - let noOutputs = true; - while (true) { - let {value, done} = await reader.read(); - value = new TextDecoder().decode(value); - if (done) { - progressSectionEle.style.display = 'none'; - break; - } - - for (let event of value.split('\n').filter(e => e !== '')) { - const data = JSON.parse(event); - - if (data.event === 'result') { - noOutputs = false; - appendOutput(data.url, data.seed, data.config); - progressEle.setAttribute('value', 0); - progressEle.setAttribute('max', totalSteps); - } else if (data.event === 'upscaling-started') { - document.getElementById("processing_cnt").textContent=data.processed_file_cnt; - document.getElementById("scaling-inprocess-message").style.display = "block"; - } else if (data.event === 'upscaling-done') { - document.getElementById("scaling-inprocess-message").style.display = "none"; - } else if (data.event === 'step') { - progressEle.setAttribute('value', data.step); - if (data.url) { - progressImageEle.src = data.url; - } - } else if (data.event === 'canceled') { - // avoid alerting as if this were an error case - noOutputs = false; - } - } - } - - // Re-enable form, remove no-results-message - form.querySelector('fieldset').removeAttribute('disabled'); - document.querySelector("#prompt").value = prompt; - document.querySelector('progress').setAttribute('value', '0'); - - if (noOutputs) { - alert("Error occurred while generating."); - } + // Evaluate all checkboxes + let checkboxes = form.querySelectorAll('input[type=checkbox]'); + checkboxes.forEach(function (checkbox) { + if (checkbox.checked) { + formData[checkbox.name] = 'true'; + } + }); + + let strength = formData.strength; + let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; + let showProgressImages = formData.progress_images; + + // Set enabling flags + + + // Initialize the progress bar + initProgress(totalSteps, showProgressImages); + + // POST, use response to listen for events + fetch(form.action, { + method: form.method, + headers: new Headers({'content-type': 'application/json'}), + body: JSON.stringify(formData), + }) + .then(response => response.json()) + .then(data => { + var jobId = data.jobId; + socket.emit('join_room', { 'room': jobId }); }); - // Disable form while generating form.querySelector('fieldset').setAttribute('disabled',''); - document.querySelector("#prompt").value = `Generating: "${prompt}"`; } -async function fetchRunLog() { - try { - let response = await fetch('/run_log.json') - const data = await response.json(); - for(let item of data.run_log) { - appendOutput(item.url, item.seed, item); - } - } catch (e) { - console.error(e); - } +function fieldSetEnableChecked(event) { + cb = event.target; + fields = cb.closest('fieldset'); + fields.disabled = !cb.checked; } +// Socket listeners +socket.on('job_started', (data) => {}) + +socket.on('dream_result', (data) => { + var jobId = data.jobId; + var dreamId = data.dreamId; + var dreamRequest = data.dreamRequest; + var src = 'api/images/' + dreamId; + + priorResultsLoadState.offset += 1; + appendOutput(src, dreamRequest.seed, dreamRequest); + + resetProgress(false); +}) + +socket.on('dream_progress', (data) => { + // TODO: it'd be nice if we could get a seed reported here, but the generator would need to be updated + var step = data.step; + var totalSteps = data.totalSteps; + var jobId = data.jobId; + var dreamId = data.dreamId; + + var progressType = data.progressType + if (progressType === 'GENERATION') { + var src = data.hasProgressImage ? + 'api/intermediates/' + dreamId + '/' + step + : null; + setProgress(step, totalSteps, src); + } else if (progressType === 'UPSCALING_STARTED') { + // step and totalSteps are used for upscale count on this message + document.getElementById("processing_cnt").textContent = step; + document.getElementById("processing_total").textContent = totalSteps; + document.getElementById("scaling-inprocess-message").style.display = "block"; + } else if (progressType == 'UPSCALING_DONE') { + document.getElementById("scaling-inprocess-message").style.display = "none"; + } +}) + +socket.on('job_canceled', (data) => { + resetForm(); + resetProgress(); +}) + +socket.on('job_done', (data) => { + jobId = data.jobId + socket.emit('leave_room', { 'room': jobId }); + + resetForm(); + resetProgress(); +}) + window.onload = async () => { document.querySelector("#prompt").addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.shiftKey) { @@ -183,7 +320,7 @@ window.onload = async () => { saveFields(e.target.form); }); document.querySelector("#reset-seed").addEventListener('click', (e) => { - document.querySelector("#seed").value = -1; + document.querySelector("#seed").value = 0; saveFields(e.target.form); }); document.querySelector("#reset-all").addEventListener('click', (e) => { @@ -195,13 +332,13 @@ window.onload = async () => { loadFields(document.querySelector("#generate-form")); document.querySelector('#cancel-button').addEventListener('click', () => { - fetch('/cancel').catch(e => { + fetch('/api/cancel').catch(e => { console.error(e); }); }); document.documentElement.addEventListener('keydown', (e) => { if (e.key === "Escape") - fetch('/cancel').catch(err => { + fetch('/api/cancel').catch(err => { console.error(err); }); }); @@ -209,5 +346,51 @@ window.onload = async () => { if (!config.gfpgan_model_exists) { document.querySelector("#gfpgan").style.display = 'none'; } - await fetchRunLog() + + window.addEventListener("scroll", () => { + if ((window.innerHeight + window.pageYOffset) >= document.body.offsetHeight) { + loadPriorResults(); + } + }); + + + + // Enable/disable forms by checkboxes + document.querySelectorAll("legend > input[type=checkbox]").forEach(function(cb) { + cb.addEventListener('change', fieldSetEnableChecked); + fieldSetEnableChecked({ target: cb}) + }); + + + // Load some of the previous results + loadPriorResults(); + + // Image drop/upload WIP + /* + let drop = document.getElementById('dropper'); + function ondrop(event) { + let dreamData = event.dataTransfer.getData('dream'); + if (dreamData) { + var dream = JSON.parse(decodeURIComponent(dreamData)); + alert(dream.dreamId); + } + }; + + function ondragenter(event) { + event.preventDefault(); + }; + + function ondragover(event) { + event.preventDefault(); + }; + + function ondragleave(event) { + + } + + drop.addEventListener('drop', ondrop); + drop.addEventListener('dragenter', ondragenter); + drop.addEventListener('dragover', ondragover); + drop.addEventListener('dragleave', ondragleave); + */ }; From b3e026aa4edce564d49c31fe301ccfae8b335b4a Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 17 Sep 2022 02:18:52 -0400 Subject: [PATCH 098/238] point legacy web server at legacy static files --- ldm/dream/server.py | 8 +- static/legacy_web/favicon.ico | Bin 0 -> 1150 bytes static/legacy_web/index.css | 143 +++++++++++++++++++++++ static/legacy_web/index.html | 126 ++++++++++++++++++++ static/legacy_web/index.js | 213 ++++++++++++++++++++++++++++++++++ 5 files changed, 486 insertions(+), 4 deletions(-) create mode 100644 static/legacy_web/favicon.ico create mode 100644 static/legacy_web/index.css create mode 100644 static/legacy_web/index.html create mode 100644 static/legacy_web/index.js diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 372d719052..9147a3180a 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -76,7 +76,7 @@ class DreamServer(BaseHTTPRequestHandler): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - with open("./static/dream_web/index.html", "rb") as content: + with open("./static/legacy_web/index.html", "rb") as content: self.wfile.write(content.read()) elif self.path == "/config.js": # unfortunately this import can't be at the top level, since that would cause a circular import @@ -94,7 +94,7 @@ class DreamServer(BaseHTTPRequestHandler): self.end_headers() output = [] - log_file = os.path.join(self.outdir, "dream_web_log.txt") + log_file = os.path.join(self.outdir, "legacy_web_log.txt") if os.path.exists(log_file): with open(log_file, "r") as log: for line in log: @@ -114,7 +114,7 @@ class DreamServer(BaseHTTPRequestHandler): else: path_dir = os.path.dirname(self.path) out_dir = os.path.realpath(self.outdir.rstrip('/')) - if self.path.startswith('/static/dream_web/'): + if self.path.startswith('/static/legacy_web/'): path = '.' + self.path elif out_dir.replace('\\', '/').endswith(path_dir): file = os.path.basename(self.path) @@ -188,7 +188,7 @@ class DreamServer(BaseHTTPRequestHandler): config['seed'] = seed # Append post_data to log, but only once! if not upscaled: - with open(os.path.join(self.outdir, "dream_web_log.txt"), "a") as log: + with open(os.path.join(self.outdir, "legacy_web_log.txt"), "a") as log: log.write(f"{path}: {json.dumps(config)}\n") self.wfile.write(bytes(json.dumps( diff --git a/static/legacy_web/favicon.ico b/static/legacy_web/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..51eb844a6a4a9d4b13e17e38b0fc915e7e97d4b5 GIT binary patch literal 1150 zcmaiy%TE(g6vi*n1a-yAr5H_2eSt+l!2}h8?$p@n=nPJTglL%pit>^TL`+1D5hx&N z)!<{Tc1e&lvO-)*Ow^TsgK$#zJKYFEA;2&@TN?6A5C9Q()1;lGF^Sd zF~GSouqjvv->jVh^vZ3gw#sUXZQHSqR>WSmwCOtUf;BK6W$k#wMKX$aiq1TKiY)i0 zVAh_I80S)!qiamC2k7>K9QPINuKnap%uv%}j+#E^Jur4AXDJpbkvT6Ctz07yN&)Z7 znrGHFe)vUp?-<1^k5RnhDB0a3h^>+{H77oj<%hM0acGw^T{k?>wWp=8-IJ2<;2zkW z55$XEACugh&R(wZ1^nba=DC(TD08@HP|IVZ?1<#7_S=$s)|_Dd@;ZI;mZvYT`CA{Y z_Vq(y{pYvZf8ANnKfH$f+a32rZ=N(I_xgGd_x}n~fRYte5_cZWQRBiY+1KuqaiB`D zuiiy$g`D(znbUIcklw#ZXiGqz&xFs + + Stable Diffusion Dream Server + + + + + + + + +
+

Stable Diffusion Dream Server

+
+ For news and support for this web service, visit our GitHub site +
+
+ +
+
+
+ +
+
+ + + + + + + + + + +
+ + + + + + + + + + +
+ + + + +
+
+ + + +
+ + + + +
+
+
+ + + + + + +
+
+
+
+
+ + +
+ +
+ Postprocessing...1/3 +
+
+
+ +
+
+

No results...

+
+
+
+ + diff --git a/static/legacy_web/index.js b/static/legacy_web/index.js new file mode 100644 index 0000000000..ac68034920 --- /dev/null +++ b/static/legacy_web/index.js @@ -0,0 +1,213 @@ +function toBase64(file) { + return new Promise((resolve, reject) => { + const r = new FileReader(); + r.readAsDataURL(file); + r.onload = () => resolve(r.result); + r.onerror = (error) => reject(error); + }); +} + +function appendOutput(src, seed, config) { + let outputNode = document.createElement("figure"); + + let variations = config.with_variations; + if (config.variation_amount > 0) { + variations = (variations ? variations + ',' : '') + seed + ':' + config.variation_amount; + } + let baseseed = (config.with_variations || config.variation_amount > 0) ? config.seed : seed; + let altText = baseseed + ' | ' + (variations ? variations + ' | ' : '') + config.prompt; + + // img needs width and height for lazy loading to work + const figureContents = ` + + ${altText} + +
${seed}
+ `; + + outputNode.innerHTML = figureContents; + let figcaption = outputNode.querySelector('figcaption'); + + // Reload image config + figcaption.addEventListener('click', () => { + let form = document.querySelector("#generate-form"); + for (const [k, v] of new FormData(form)) { + if (k == 'initimg') { continue; } + form.querySelector(`*[name=${k}]`).value = config[k]; + } + + document.querySelector("#seed").value = baseseed; + document.querySelector("#with_variations").value = variations || ''; + if (document.querySelector("#variation_amount").value <= 0) { + document.querySelector("#variation_amount").value = 0.2; + } + + saveFields(document.querySelector("#generate-form")); + }); + + document.querySelector("#results").prepend(outputNode); +} + +function saveFields(form) { + for (const [k, v] of new FormData(form)) { + if (typeof v !== 'object') { // Don't save 'file' type + localStorage.setItem(k, v); + } + } +} + +function loadFields(form) { + for (const [k, v] of new FormData(form)) { + const item = localStorage.getItem(k); + if (item != null) { + form.querySelector(`*[name=${k}]`).value = item; + } + } +} + +function clearFields(form) { + localStorage.clear(); + let prompt = form.prompt.value; + form.reset(); + form.prompt.value = prompt; +} + +const BLANK_IMAGE_URL = 'data:image/svg+xml,'; +async function generateSubmit(form) { + const prompt = document.querySelector("#prompt").value; + + // Convert file data to base64 + let formData = Object.fromEntries(new FormData(form)); + formData.initimg_name = formData.initimg.name + formData.initimg = formData.initimg.name !== '' ? await toBase64(formData.initimg) : null; + + let strength = formData.strength; + let totalSteps = formData.initimg ? Math.floor(strength * formData.steps) : formData.steps; + + let progressSectionEle = document.querySelector('#progress-section'); + progressSectionEle.style.display = 'initial'; + let progressEle = document.querySelector('#progress-bar'); + progressEle.setAttribute('max', totalSteps); + let progressImageEle = document.querySelector('#progress-image'); + progressImageEle.src = BLANK_IMAGE_URL; + + progressImageEle.style.display = {}.hasOwnProperty.call(formData, 'progress_images') ? 'initial': 'none'; + + // Post as JSON, using Fetch streaming to get results + fetch(form.action, { + method: form.method, + body: JSON.stringify(formData), + }).then(async (response) => { + const reader = response.body.getReader(); + + let noOutputs = true; + while (true) { + let {value, done} = await reader.read(); + value = new TextDecoder().decode(value); + if (done) { + progressSectionEle.style.display = 'none'; + break; + } + + for (let event of value.split('\n').filter(e => e !== '')) { + const data = JSON.parse(event); + + if (data.event === 'result') { + noOutputs = false; + appendOutput(data.url, data.seed, data.config); + progressEle.setAttribute('value', 0); + progressEle.setAttribute('max', totalSteps); + } else if (data.event === 'upscaling-started') { + document.getElementById("processing_cnt").textContent=data.processed_file_cnt; + document.getElementById("scaling-inprocess-message").style.display = "block"; + } else if (data.event === 'upscaling-done') { + document.getElementById("scaling-inprocess-message").style.display = "none"; + } else if (data.event === 'step') { + progressEle.setAttribute('value', data.step); + if (data.url) { + progressImageEle.src = data.url; + } + } else if (data.event === 'canceled') { + // avoid alerting as if this were an error case + noOutputs = false; + } + } + } + + // Re-enable form, remove no-results-message + form.querySelector('fieldset').removeAttribute('disabled'); + document.querySelector("#prompt").value = prompt; + document.querySelector('progress').setAttribute('value', '0'); + + if (noOutputs) { + alert("Error occurred while generating."); + } + }); + + // Disable form while generating + form.querySelector('fieldset').setAttribute('disabled',''); + document.querySelector("#prompt").value = `Generating: "${prompt}"`; +} + +async function fetchRunLog() { + try { + let response = await fetch('/run_log.json') + const data = await response.json(); + for(let item of data.run_log) { + appendOutput(item.url, item.seed, item); + } + } catch (e) { + console.error(e); + } +} + +window.onload = async () => { + document.querySelector("#prompt").addEventListener("keydown", (e) => { + if (e.key === "Enter" && !e.shiftKey) { + const form = e.target.form; + generateSubmit(form); + } + }); + document.querySelector("#generate-form").addEventListener('submit', (e) => { + e.preventDefault(); + const form = e.target; + + generateSubmit(form); + }); + document.querySelector("#generate-form").addEventListener('change', (e) => { + saveFields(e.target.form); + }); + document.querySelector("#reset-seed").addEventListener('click', (e) => { + document.querySelector("#seed").value = -1; + saveFields(e.target.form); + }); + document.querySelector("#reset-all").addEventListener('click', (e) => { + clearFields(e.target.form); + }); + document.querySelector("#remove-image").addEventListener('click', (e) => { + initimg.value=null; + }); + loadFields(document.querySelector("#generate-form")); + + document.querySelector('#cancel-button').addEventListener('click', () => { + fetch('/cancel').catch(e => { + console.error(e); + }); + }); + document.documentElement.addEventListener('keydown', (e) => { + if (e.key === "Escape") + fetch('/cancel').catch(err => { + console.error(err); + }); + }); + + if (!config.gfpgan_model_exists) { + document.querySelector("#gfpgan").style.display = 'none'; + } + await fetchRunLog() +}; From e45f46d6731a6499871583bfa02a1739d969f06e Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat, 17 Sep 2022 16:32:59 +1000 Subject: [PATCH 099/238] Improves code structure, comments, formatting, linting --- frontend/README.md | 88 +-- frontend/dist/assets/index.3d2e59c5.js | 694 +++++++++++++++++ frontend/dist/assets/index.cc5cde43.js | 695 ------------------ frontend/dist/index.html | 2 +- frontend/package.json | 3 +- frontend/src/App.tsx | 85 +-- frontend/src/app/parameterTranslation.ts | 8 +- frontend/src/components/SDButton.tsx | 19 +- frontend/src/components/SDNumberInput.tsx | 7 +- frontend/src/components/SDSelect.tsx | 95 ++- frontend/src/components/SDSwitch.tsx | 5 +- .../src/features/gallery/CurrentImage.tsx | 161 ---- .../features/gallery/CurrentImageButtons.tsx | 141 ++++ .../features/gallery/CurrentImageDisplay.tsx | 67 ++ .../src/features/gallery/DeleteImageModal.tsx | 121 +++ .../gallery/DeleteImageModalButton.tsx | 94 --- .../src/features/gallery/HoverableImage.tsx | 131 ++++ .../src/features/gallery/ImageGallery.tsx | 35 + .../features/gallery/ImageMetadataViewer.tsx | 222 +++--- frontend/src/features/gallery/ImageRoll.tsx | 150 ---- frontend/src/features/header/SiteHeader.tsx | 44 +- frontend/src/features/sd/ESRGANOptions.tsx | 111 +-- frontend/src/features/sd/GFPGANOptions.tsx | 79 +- .../src/features/sd/ImageToImageOptions.tsx | 79 +- frontend/src/features/sd/ImageUploader.tsx | 63 ++ .../{InitImage.css => InitAndMaskImage.css} | 0 frontend/src/features/sd/InitAndMaskImage.tsx | 57 ++ .../features/sd/InitAndMaskUploadButtons.tsx | 131 ++++ frontend/src/features/sd/InitImage.tsx | 155 ---- frontend/src/features/sd/MaskUploader.tsx | 61 -- frontend/src/features/sd/OptionsAccordion.tsx | 362 ++++----- frontend/src/features/sd/OutputOptions.tsx | 94 +-- frontend/src/features/sd/ProcessButtons.tsx | 84 ++- frontend/src/features/sd/PromptInput.tsx | 15 +- frontend/src/features/sd/SDSlider.tsx | 51 -- frontend/src/features/sd/SamplerOptions.tsx | 88 ++- .../src/features/sd/SeedVariationOptions.tsx | 257 ++++--- frontend/src/features/sd/Variant.tsx | 92 --- frontend/src/features/sd/sdSlice.ts | 18 +- frontend/src/features/system/LogViewer.tsx | 199 +++-- .../src/features/system/SettingsModal.tsx | 278 ++++--- .../src/features/system/useCheckParameters.ts | 158 ++-- 42 files changed, 2655 insertions(+), 2644 deletions(-) create mode 100644 frontend/dist/assets/index.3d2e59c5.js delete mode 100644 frontend/dist/assets/index.cc5cde43.js delete mode 100644 frontend/src/features/gallery/CurrentImage.tsx create mode 100644 frontend/src/features/gallery/CurrentImageButtons.tsx create mode 100644 frontend/src/features/gallery/CurrentImageDisplay.tsx create mode 100644 frontend/src/features/gallery/DeleteImageModal.tsx delete mode 100644 frontend/src/features/gallery/DeleteImageModalButton.tsx create mode 100644 frontend/src/features/gallery/HoverableImage.tsx create mode 100644 frontend/src/features/gallery/ImageGallery.tsx delete mode 100644 frontend/src/features/gallery/ImageRoll.tsx create mode 100644 frontend/src/features/sd/ImageUploader.tsx rename frontend/src/features/sd/{InitImage.css => InitAndMaskImage.css} (100%) create mode 100644 frontend/src/features/sd/InitAndMaskImage.tsx create mode 100644 frontend/src/features/sd/InitAndMaskUploadButtons.tsx delete mode 100644 frontend/src/features/sd/InitImage.tsx delete mode 100644 frontend/src/features/sd/MaskUploader.tsx delete mode 100644 frontend/src/features/sd/SDSlider.tsx delete mode 100644 frontend/src/features/sd/Variant.tsx diff --git a/frontend/README.md b/frontend/README.md index 6928e27b49..a611738808 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,85 +1,25 @@ # Stable Diffusion Web UI -Demo at https://peaceful-otter-7a427f.netlify.app/ (not connected to back end) - -much of this readme is just notes for myself during dev work - -numpy rand: 0 to 4294967295 - -## Test and Build +## Build from `frontend/`: -- `yarn dev` runs `tsc-watch`, which runs `vite build` on successful `tsc` transpilation +- `yarn dev` runs vite dev server +- `yarn build-dev` builds dev +- `yarn build` builds prod from `.`: -- `python backend/server.py` serves both frontend and backend at http://localhost:9090 - -## API - -`backend/server.py` serves the UI and provides a [socket.io](https://github.com/socketio/socket.io) API via [flask-socketio](https://github.com/miguelgrinberg/flask-socketio). - -### Server Listeners - -The server listens for these socket.io events: - -`cancel` - -- Cancels in-progress image generation -- Returns ack only - -`generateImage` - -- Accepts object of image parameters -- Generates an image -- Returns ack only (image generation function sends progress and result via separate events) - -`deleteImage` - -- Accepts file path to image -- Deletes image -- Returns ack only - -`deleteAllImages` WIP - -- Deletes all images in `outputs/` -- Returns ack only - -`requestAllImages` - -- Returns array of all images in `outputs/` - -`requestCapabilities` WIP - -- Returns capabilities of server (torch device, GFPGAN and ESRGAN availability, ???) - -`sendImage` WIP - -- Accepts a File and attributes -- Saves image -- Used to save init images which are not generated images - -### Server Emitters - -`progress` - -- Emitted during each step in generation -- Sends a number from 0 to 1 representing percentage of steps completed - -`result` WIP - -- Emitted when an image generation has completed -- Sends a object: - -``` -{ - url: relative_file_path, - metadata: image_metadata_object -} -``` +- `python backend/server.py` serves both frontend and backend at http://localhost:9090 ## TODO -- Search repo for "TODO" -- My one gripe with Chakra: no way to disable all animations right now and drop the dependence on `framer-motion`. I would prefer to save the ~30kb on bundle and have zero animations. This is on the Chakra roadmap. See https://github.com/chakra-ui/chakra-ui/pull/6368 for last discussion on this. Need to check in on this issue periodically. +- Search repo for "TODO" +- My one gripe with Chakra: no way to disable all animations right now and drop the dependence on + `framer-motion`. I would prefer to save the ~30kb on bundle and have zero animations. This is on + the Chakra roadmap. See https://github.com/chakra-ui/chakra-ui/pull/6368 for last discussion on + this. Need to check in on this issue periodically. +- More status info e.g. phase of processing, image we are on of the total count, etc +- Mobile friendly layout +- Proper image gallery/viewer/manager +- Instead of deleting images directly, use something like [send2trash](https://pypi.org/project/Send2Trash/) diff --git a/frontend/dist/assets/index.3d2e59c5.js b/frontend/dist/assets/index.3d2e59c5.js new file mode 100644 index 0000000000..ca609e8fa2 --- /dev/null +++ b/frontend/dist/assets/index.3d2e59c5.js @@ -0,0 +1,694 @@ +function WV(e,t){for(var r=0;ri[s]})}}}return Object.freeze(Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}))}(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const s of document.querySelectorAll('link[rel="modulepreload"]'))i(s);new MutationObserver(s=>{for(const c of s)if(c.type==="childList")for(const u of c.addedNodes)u.tagName==="LINK"&&u.rel==="modulepreload"&&i(u)}).observe(document,{childList:!0,subtree:!0});function r(s){const c={};return s.integrity&&(c.integrity=s.integrity),s.referrerpolicy&&(c.referrerPolicy=s.referrerpolicy),s.crossorigin==="use-credentials"?c.credentials="include":s.crossorigin==="anonymous"?c.credentials="omit":c.credentials="same-origin",c}function i(s){if(s.ep)return;s.ep=!0;const c=r(s);fetch(s.href,c)}})();var lc=typeof globalThis<"u"?globalThis:typeof window<"u"?window:typeof global<"u"?global:typeof self<"u"?self:{};function GV(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var D={exports:{}},OA={exports:{}};/** + * @license React + * react.development.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */(function(e,t){(function(){typeof __REACT_DEVTOOLS_GLOBAL_HOOK__<"u"&&typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStart=="function"&&__REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStart(new Error);var r="18.2.0",i=Symbol.for("react.element"),s=Symbol.for("react.portal"),c=Symbol.for("react.fragment"),u=Symbol.for("react.strict_mode"),h=Symbol.for("react.profiler"),m=Symbol.for("react.provider"),v=Symbol.for("react.context"),b=Symbol.for("react.forward_ref"),x=Symbol.for("react.suspense"),C=Symbol.for("react.suspense_list"),_=Symbol.for("react.memo"),N=Symbol.for("react.lazy"),A=Symbol.for("react.offscreen"),I=Symbol.iterator,j="@@iterator";function M(S){if(S===null||typeof S!="object")return null;var R=I&&S[I]||S[j];return typeof R=="function"?R:null}var z={current:null},U={transition:null},V={current:null,isBatchingLegacy:!1,didScheduleLegacyUpdate:!1},K={current:null},X={},de=null;function ve(S){de=S}X.setExtraStackFrame=function(S){de=S},X.getCurrentStack=null,X.getStackAddendum=function(){var S="";de&&(S+=de);var R=X.getCurrentStack;return R&&(S+=R()||""),S};var se=!1,be=!1,Xe=!1,ie=!1,ce=!1,Re={ReactCurrentDispatcher:z,ReactCurrentBatchConfig:U,ReactCurrentOwner:K};Re.ReactDebugCurrentFrame=X,Re.ReactCurrentActQueue=V;function Ce(S){{for(var R=arguments.length,$=new Array(R>1?R-1:0),G=1;G1?R-1:0),G=1;G1){for(var Nt=Array(gt),pt=0;pt1){for(var Lt=Array(pt),wt=0;wt is not supported and will be removed in a future major release. Did you mean to render instead?")),R.Provider},set:function(xe){R.Provider=xe}},_currentValue:{get:function(){return R._currentValue},set:function(xe){R._currentValue=xe}},_currentValue2:{get:function(){return R._currentValue2},set:function(xe){R._currentValue2=xe}},_threadCount:{get:function(){return R._threadCount},set:function(xe){R._threadCount=xe}},Consumer:{get:function(){return $||($=!0,oe("Rendering is not supported and will be removed in a future major release. Did you mean to render instead?")),R.Consumer}},displayName:{get:function(){return R.displayName},set:function(xe){te||(Ce("Setting `displayName` on Context.Consumer has no effect. You should set it directly on the context with Context.displayName = '%s'.",xe),te=!0)}}}),R.Consumer=Fe}return R._currentRenderer=null,R._currentRenderer2=null,R}var cr=-1,Oa=0,Di=1,ka=2;function q(S){if(S._status===cr){var R=S._result,$=R();if($.then(function(Fe){if(S._status===Oa||S._status===cr){var xe=S;xe._status=Di,xe._result=Fe}},function(Fe){if(S._status===Oa||S._status===cr){var xe=S;xe._status=ka,xe._result=Fe}}),S._status===cr){var G=S;G._status=Oa,G._result=$}}if(S._status===Di){var te=S._result;return te===void 0&&oe(`lazy: Expected the result of a dynamic import() call. Instead received: %s + +Your code should look like: + const MyComponent = lazy(() => import('./MyComponent')) + +Did you accidentally put curly braces around the import?`,te),"default"in te||oe(`lazy: Expected the result of a dynamic import() call. Instead received: %s + +Your code should look like: + const MyComponent = lazy(() => import('./MyComponent'))`,te),te.default}else throw S._result}function Be(S){var R={_status:cr,_result:S},$={$$typeof:N,_payload:R,_init:q};{var G,te;Object.defineProperties($,{defaultProps:{configurable:!0,get:function(){return G},set:function(Fe){oe("React.lazy(...): It is not supported to assign `defaultProps` to a lazy component import. Either specify them where the component is defined, or create a wrapping component around it."),G=Fe,Object.defineProperty($,"defaultProps",{enumerable:!0})}},propTypes:{configurable:!0,get:function(){return te},set:function(Fe){oe("React.lazy(...): It is not supported to assign `propTypes` to a lazy component import. Either specify them where the component is defined, or create a wrapping component around it."),te=Fe,Object.defineProperty($,"propTypes",{enumerable:!0})}}})}return $}function qe(S){S!=null&&S.$$typeof===_?oe("forwardRef requires a render function but received a `memo` component. Instead of forwardRef(memo(...)), use memo(forwardRef(...))."):typeof S!="function"?oe("forwardRef requires a render function but was given %s.",S===null?"null":typeof S):S.length!==0&&S.length!==2&&oe("forwardRef render functions accept exactly two parameters: props and ref. %s",S.length===1?"Did you forget to use the ref parameter?":"Any additional parameter will be undefined."),S!=null&&(S.defaultProps!=null||S.propTypes!=null)&&oe("forwardRef render functions do not support propTypes or defaultProps. Did you accidentally pass a React component?");var R={$$typeof:b,render:S};{var $;Object.defineProperty(R,"displayName",{enumerable:!1,configurable:!0,get:function(){return $},set:function(G){$=G,!S.name&&!S.displayName&&(S.displayName=G)}})}return R}var bt;bt=Symbol.for("react.module.reference");function nn(S){return!!(typeof S=="string"||typeof S=="function"||S===c||S===h||ce||S===u||S===x||S===C||ie||S===A||se||be||Xe||typeof S=="object"&&S!==null&&(S.$$typeof===N||S.$$typeof===_||S.$$typeof===m||S.$$typeof===v||S.$$typeof===b||S.$$typeof===bt||S.getModuleId!==void 0))}function gn(S,R){nn(S)||oe("memo: The first argument must be a component. Instead received: %s",S===null?"null":typeof S);var $={$$typeof:_,type:S,compare:R===void 0?null:R};{var G;Object.defineProperty($,"displayName",{enumerable:!1,configurable:!0,get:function(){return G},set:function(te){G=te,!S.name&&!S.displayName&&(S.displayName=te)}})}return $}function tt(){var S=z.current;return S===null&&oe(`Invalid hook call. Hooks can only be called inside of the body of a function component. This could happen for one of the following reasons: +1. You might have mismatching versions of React and the renderer (such as React DOM) +2. You might be breaking the Rules of Hooks +3. You might have more than one copy of React in the same app +See https://reactjs.org/link/invalid-hook-call for tips about how to debug and fix this problem.`),S}function $t(S){var R=tt();if(S._context!==void 0){var $=S._context;$.Consumer===S?oe("Calling useContext(Context.Consumer) is not supported, may cause bugs, and will be removed in a future major release. Did you mean to call useContext(Context) instead?"):$.Provider===S&&oe("Calling useContext(Context.Provider) is not supported. Did you mean to call useContext(Context) instead?")}return R.useContext(S)}function zn(S){var R=tt();return R.useState(S)}function In(S,R,$){var G=tt();return G.useReducer(S,R,$)}function an(S){var R=tt();return R.useRef(S)}function Lr(S,R){var $=tt();return $.useEffect(S,R)}function ci(S,R){var $=tt();return $.useInsertionEffect(S,R)}function _o(S,R){var $=tt();return $.useLayoutEffect(S,R)}function da(S,R){var $=tt();return $.useCallback(S,R)}function Qi(S,R){var $=tt();return $.useMemo(S,R)}function yu(S,R,$){var G=tt();return G.useImperativeHandle(S,R,$)}function fi(S,R){{var $=tt();return $.useDebugValue(S,R)}}function zs(){var S=tt();return S.useTransition()}function Pi(S){var R=tt();return R.useDeferredValue(S)}function Zt(){var S=tt();return S.useId()}function Mi(S,R,$){var G=tt();return G.useSyncExternalStore(S,R,$)}var pa=0,Eo,es,To,ts,ns,No,Ro;function rs(){}rs.__reactDisabledLog=!0;function Bs(){{if(pa===0){Eo=console.log,es=console.info,To=console.warn,ts=console.error,ns=console.group,No=console.groupCollapsed,Ro=console.groupEnd;var S={configurable:!0,enumerable:!0,value:rs,writable:!0};Object.defineProperties(console,{info:S,log:S,warn:S,error:S,group:S,groupCollapsed:S,groupEnd:S})}pa++}}function Us(){{if(pa--,pa===0){var S={configurable:!0,enumerable:!0,writable:!0};Object.defineProperties(console,{log:Ie({},S,{value:Eo}),info:Ie({},S,{value:es}),warn:Ie({},S,{value:To}),error:Ie({},S,{value:ts}),group:Ie({},S,{value:ns}),groupCollapsed:Ie({},S,{value:No}),groupEnd:Ie({},S,{value:Ro})})}pa<0&&oe("disabledDepth fell below zero. This is a bug in React. Please file an issue.")}}var di=Re.ReactCurrentDispatcher,Dr;function Da(S,R,$){{if(Dr===void 0)try{throw Error()}catch(te){var G=te.stack.trim().match(/\n( *(at )?)/);Dr=G&&G[1]||""}return` +`+Dr+S}}var ha=!1,Pa;{var as=typeof WeakMap=="function"?WeakMap:Map;Pa=new as}function Ao(S,R){if(!S||ha)return"";{var $=Pa.get(S);if($!==void 0)return $}var G;ha=!0;var te=Error.prepareStackTrace;Error.prepareStackTrace=void 0;var Fe;Fe=di.current,di.current=null,Bs();try{if(R){var xe=function(){throw Error()};if(Object.defineProperty(xe.prototype,"props",{set:function(){throw Error()}}),typeof Reflect=="object"&&Reflect.construct){try{Reflect.construct(xe,[])}catch(Dt){G=Dt}Reflect.construct(S,[],xe)}else{try{xe.call()}catch(Dt){G=Dt}S.call(xe.prototype)}}else{try{throw Error()}catch(Dt){G=Dt}S()}}catch(Dt){if(Dt&&G&&typeof Dt.stack=="string"){for(var $e=Dt.stack.split(` +`),it=G.stack.split(` +`),gt=$e.length-1,Nt=it.length-1;gt>=1&&Nt>=0&&$e[gt]!==it[Nt];)Nt--;for(;gt>=1&&Nt>=0;gt--,Nt--)if($e[gt]!==it[Nt]){if(gt!==1||Nt!==1)do if(gt--,Nt--,Nt<0||$e[gt]!==it[Nt]){var pt=` +`+$e[gt].replace(" at new "," at ");return S.displayName&&pt.includes("")&&(pt=pt.replace("",S.displayName)),typeof S=="function"&&Pa.set(S,pt),pt}while(gt>=1&&Nt>=0);break}}}finally{ha=!1,di.current=Fe,Us(),Error.prepareStackTrace=te}var Lt=S?S.displayName||S.name:"",wt=Lt?Da(Lt):"";return typeof S=="function"&&Pa.set(S,wt),wt}function is(S,R,$){return Ao(S,!1)}function Nl(S){var R=S.prototype;return!!(R&&R.isReactComponent)}function ma(S,R,$){if(S==null)return"";if(typeof S=="function")return Ao(S,Nl(S));if(typeof S=="string")return Da(S);switch(S){case x:return Da("Suspense");case C:return Da("SuspenseList")}if(typeof S=="object")switch(S.$$typeof){case b:return is(S.render);case _:return ma(S.type,R,$);case N:{var G=S,te=G._payload,Fe=G._init;try{return ma(Fe(te),R,$)}catch{}}}return""}var Oo={},Ma=Re.ReactDebugCurrentFrame;function pi(S){if(S){var R=S._owner,$=ma(S.type,S._source,R?R.type:null);Ma.setExtraStackFrame($)}else Ma.setExtraStackFrame(null)}function js(S,R,$,G,te){{var Fe=Function.call.bind(fn);for(var xe in S)if(Fe(S,xe)){var $e=void 0;try{if(typeof S[xe]!="function"){var it=Error((G||"React class")+": "+$+" type `"+xe+"` is invalid; it must be a function, usually from the `prop-types` package, but received `"+typeof S[xe]+"`.This often happens because of typos such as `PropTypes.function` instead of `PropTypes.func`.");throw it.name="Invariant Violation",it}$e=S[xe](R,xe,G,$,null,"SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED")}catch(gt){$e=gt}$e&&!($e instanceof Error)&&(pi(te),oe("%s: type specification of %s `%s` is invalid; the type checker function must return `null` or an `Error` but returned a %s. You may have forgotten to pass an argument to the type checker creator (arrayOf, instanceOf, objectOf, oneOf, oneOfType, and shape all require an argument).",G||"React class",$,xe,typeof $e),pi(null)),$e instanceof Error&&!($e.message in Oo)&&(Oo[$e.message]=!0,pi(te),oe("Failed %s type: %s",$,$e.message),pi(null))}}}function on(S){if(S){var R=S._owner,$=ma(S.type,S._source,R?R.type:null);ve($)}else ve(null)}var hi;hi=!1;function ko(){if(K.current){var S=Bt(K.current.type);if(S)return` + +Check the render method of \``+S+"`."}return""}function It(S){if(S!==void 0){var R=S.fileName.replace(/^.*[\\\/]/,""),$=S.lineNumber;return` + +Check your code at `+R+":"+$+"."}return""}function $s(S){return S!=null?It(S.__source):""}var yr={};function Ii(S){var R=ko();if(!R){var $=typeof S=="string"?S:S.displayName||S.name;$&&(R=` + +Check the top-level render call using <`+$+">.")}return R}function ja(S,R){if(!(!S._store||S._store.validated||S.key!=null)){S._store.validated=!0;var $=Ii(R);if(!yr[$]){yr[$]=!0;var G="";S&&S._owner&&S._owner!==K.current&&(G=" It was passed a child from "+Bt(S._owner.type)+"."),on(S),oe('Each child in a list should have a unique "key" prop.%s%s See https://reactjs.org/link/warning-keys for more information.',$,G),on(null)}}}function Ji(S,R){if(typeof S=="object"){if(Yt(S))for(var $=0;$",te=" Did you accidentally export a JSX literal instead of a component?"):xe=typeof S,oe("React.createElement: type is invalid -- expected a string (for built-in components) or a class/function (for composite components) but got: %s.%s",xe,te)}var $e=rt.apply(this,arguments);if($e==null)return $e;if(G)for(var it=2;it10&&Ce("Detected a large number of updates inside startTransition. If this is due to a subscription please re-write it to use React provided hooks. Otherwise concurrent mode guarantees are off the table."),G._updatedFibers.clear()}}}var eo=!1,mi=null;function Vs(S){if(mi===null)try{var R=("require"+Math.random()).slice(0,7),$=e&&e[R];mi=$.call(e,"timers").setImmediate}catch{mi=function(te){eo===!1&&(eo=!0,typeof MessageChannel>"u"&&oe("This browser does not have a MessageChannel implementation, so enqueuing tasks via await act(async () => ...) will fail. Please file an issue at https://github.com/facebook/react/issues if you encounter this warning."));var Fe=new MessageChannel;Fe.port1.onmessage=te,Fe.port2.postMessage(void 0)}}return mi(S)}var pn=0,Pn=!1;function Rl(S){{var R=pn;pn++,V.current===null&&(V.current=[]);var $=V.isBatchingLegacy,G;try{if(V.isBatchingLegacy=!0,G=S(),!$&&V.didScheduleLegacyUpdate){var te=V.current;te!==null&&(V.didScheduleLegacyUpdate=!1,fe(te))}}catch(Lt){throw Ia(R),Lt}finally{V.isBatchingLegacy=$}if(G!==null&&typeof G=="object"&&typeof G.then=="function"){var Fe=G,xe=!1,$e={then:function(Lt,wt){xe=!0,Fe.then(function(Dt){Ia(R),pn===0?W(Dt,Lt,wt):Lt(Dt)},function(Dt){Ia(R),wt(Dt)})}};return!Pn&&typeof Promise<"u"&&Promise.resolve().then(function(){}).then(function(){xe||(Pn=!0,oe("You called act(async () => ...) without await. This could lead to unexpected testing behaviour, interleaving multiple act calls and mixing their scopes. You should - await act(async () => ...);"))}),$e}else{var it=G;if(Ia(R),pn===0){var gt=V.current;gt!==null&&(fe(gt),V.current=null);var Nt={then:function(Lt,wt){V.current===null?(V.current=[],W(it,Lt,wt)):Lt(it)}};return Nt}else{var pt={then:function(Lt,wt){Lt(it)}};return pt}}}}function Ia(S){S!==pn-1&&oe("You seem to have overlapping act() calls, this is not supported. Be sure to await previous act() calls before making a new one. "),pn=S}function W(S,R,$){{var G=V.current;if(G!==null)try{fe(G),Vs(function(){G.length===0?(V.current=null,R(S)):W(S,R,$)})}catch(te){$(te)}else R(S)}}var Q=!1;function fe(S){if(!Q){Q=!0;var R=0;try{for(;R0;){var Xt=dn-1>>>1,wn=je[Xt];if(v(wn,rt)>0)je[Xt]=rt,je[dn]=wn,dn=Xt;else return}}function m(je,rt,xt){for(var dn=xt,Xt=je.length,wn=Xt>>>1;dnxt&&(!je||ir()));){var dn=ie.callback;if(typeof dn=="function"){ie.callback=null,ce=ie.priorityLevel;var Xt=ie.expirationTime<=xt,wn=dn(Xt);xt=e.unstable_now(),typeof wn=="function"?ie.callback=wn:ie===c(se)&&u(se),we(xt)}else u(se);ie=c(se)}if(ie!==null)return!0;var Dn=c(be);return Dn!==null&&Tt(Ie,Dn.startTime-xt),!1}function st(je,rt){switch(je){case b:case x:case C:case _:case N:break;default:je=C}var xt=ce;ce=je;try{return rt()}finally{ce=xt}}function mt(je){var rt;switch(ce){case b:case x:case C:rt=C;break;default:rt=ce;break}var xt=ce;ce=rt;try{return je()}finally{ce=xt}}function Gt(je){var rt=ce;return function(){var xt=ce;ce=rt;try{return je.apply(this,arguments)}finally{ce=xt}}}function Qe(je,rt,xt){var dn=e.unstable_now(),Xt;if(typeof xt=="object"&&xt!==null){var wn=xt.delay;typeof wn=="number"&&wn>0?Xt=dn+wn:Xt=dn}else Xt=dn;var Dn;switch(je){case b:Dn=V;break;case x:Dn=K;break;case N:Dn=ve;break;case _:Dn=de;break;case C:default:Dn=X;break}var Nr=Xt+Dn,Tn={id:Xe++,callback:rt,priorityLevel:je,startTime:Xt,expirationTime:Nr,sortIndex:-1};return Xt>dn?(Tn.sortIndex=Xt,s(be,Tn),c(se)===null&&Tn===c(be)&&(oe?ke():oe=!0,Tt(Ie,Xt-dn))):(Tn.sortIndex=Nr,s(se,Tn),!Ce&&!Re&&(Ce=!0,Jt(Le))),Tn}function vt(){}function Et(){!Ce&&!Re&&(Ce=!0,Jt(Le))}function zt(){return c(se)}function We(je){je.callback=null}function Yt(){return ce}var ye=!1,Mt=null,Kt=-1,St=i,ar=-1;function ir(){var je=e.unstable_now()-ar;return!(je125){console.error("forceFrameRate takes a positive int between 0 and 125, forcing frame rates higher than 125 fps is not supported");return}je>0?St=Math.floor(1e3/je):St=i}var kn=function(){if(Mt!==null){var je=e.unstable_now();ar=je;var rt=!0,xt=!0;try{xt=Mt(rt,je)}finally{xt?hn():(ye=!1,Mt=null)}}else ye=!1},hn;if(typeof pe=="function")hn=function(){pe(kn)};else if(typeof MessageChannel<"u"){var Ve=new MessageChannel,Ze=Ve.port2;Ve.port1.onmessage=kn,hn=function(){Ze.postMessage(null)}}else hn=function(){Se(kn,0)};function Jt(je){Mt=je,ye||(ye=!0,hn())}function Tt(je,rt){Kt=Se(function(){je(e.unstable_now())},rt)}function ke(){Te(Kt),Kt=-1}var jt=Bt,xn=null;e.unstable_IdlePriority=N,e.unstable_ImmediatePriority=b,e.unstable_LowPriority=_,e.unstable_NormalPriority=C,e.unstable_Profiling=xn,e.unstable_UserBlockingPriority=x,e.unstable_cancelCallback=We,e.unstable_continueExecution=Et,e.unstable_forceFrameRate=fn,e.unstable_getCurrentPriorityLevel=Yt,e.unstable_getFirstCallbackNode=zt,e.unstable_next=mt,e.unstable_pauseExecution=vt,e.unstable_requestPaint=jt,e.unstable_runWithPriority=st,e.unstable_scheduleCallback=Qe,e.unstable_shouldYield=ir,e.unstable_wrapCallback=Gt,typeof __REACT_DEVTOOLS_GLOBAL_HOOK__<"u"&&typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStop=="function"&&__REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStop(new Error)})()})(v3);(function(e){e.exports=v3})(m3);/** + * @license React + * react-dom.development.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */(function(){typeof __REACT_DEVTOOLS_GLOBAL_HOOK__<"u"&&typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStart=="function"&&__REACT_DEVTOOLS_GLOBAL_HOOK__.registerInternalModuleStart(new Error);var e=D.exports,t=m3.exports,r=e.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED,i=!1;function s(n){i=n}function c(n){if(!i){for(var a=arguments.length,o=new Array(a>1?a-1:0),f=1;f1?a-1:0),f=1;f2&&(n[0]==="o"||n[0]==="O")&&(n[1]==="n"||n[1]==="N")}function Nr(n,a,o,f){if(o!==null&&o.type===Ve)return!1;switch(typeof a){case"function":case"symbol":return!0;case"boolean":{if(f)return!1;if(o!==null)return!o.acceptsBooleans;var p=n.toLowerCase().slice(0,5);return p!=="data-"&&p!=="aria-"}default:return!1}}function Tn(n,a,o,f){if(a===null||typeof a>"u"||Nr(n,a,o,f))return!0;if(f)return!1;if(o!==null)switch(o.type){case Tt:return!a;case ke:return a===!1;case jt:return isNaN(a);case xn:return isNaN(a)||a<1}return!1}function ca(n){return Cn.hasOwnProperty(n)?Cn[n]:null}function Fn(n,a,o,f,p,y,w){this.acceptsBooleans=a===Jt||a===Tt||a===ke,this.attributeName=f,this.attributeNamespace=p,this.mustUseProperty=o,this.propertyName=n,this.type=a,this.sanitizeURL=y,this.removeEmptyString=w}var Cn={},fa=["children","dangerouslySetInnerHTML","defaultValue","defaultChecked","innerHTML","suppressContentEditableWarning","suppressHydrationWarning","style"];fa.forEach(function(n){Cn[n]=new Fn(n,Ve,!1,n,null,!1,!1)}),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach(function(n){var a=n[0],o=n[1];Cn[a]=new Fn(a,Ze,!1,o,null,!1,!1)}),["contentEditable","draggable","spellCheck","value"].forEach(function(n){Cn[n]=new Fn(n,Jt,!1,n.toLowerCase(),null,!1,!1)}),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach(function(n){Cn[n]=new Fn(n,Jt,!1,n,null,!1,!1)}),["allowFullScreen","async","autoFocus","autoPlay","controls","default","defer","disabled","disablePictureInPicture","disableRemotePlayback","formNoValidate","hidden","loop","noModule","noValidate","open","playsInline","readOnly","required","reversed","scoped","seamless","itemScope"].forEach(function(n){Cn[n]=new Fn(n,Tt,!1,n.toLowerCase(),null,!1,!1)}),["checked","multiple","muted","selected"].forEach(function(n){Cn[n]=new Fn(n,Tt,!0,n,null,!1,!1)}),["capture","download"].forEach(function(n){Cn[n]=new Fn(n,ke,!1,n,null,!1,!1)}),["cols","rows","size","span"].forEach(function(n){Cn[n]=new Fn(n,xn,!1,n,null,!1,!1)}),["rowSpan","start"].forEach(function(n){Cn[n]=new Fn(n,jt,!1,n.toLowerCase(),null,!1,!1)});var gr=/[\-\:]([a-z])/g,xo=function(n){return n[1].toUpperCase()};["accent-height","alignment-baseline","arabic-form","baseline-shift","cap-height","clip-path","clip-rule","color-interpolation","color-interpolation-filters","color-profile","color-rendering","dominant-baseline","enable-background","fill-opacity","fill-rule","flood-color","flood-opacity","font-family","font-size","font-size-adjust","font-stretch","font-style","font-variant","font-weight","glyph-name","glyph-orientation-horizontal","glyph-orientation-vertical","horiz-adv-x","horiz-origin-x","image-rendering","letter-spacing","lighting-color","marker-end","marker-mid","marker-start","overline-position","overline-thickness","paint-order","panose-1","pointer-events","rendering-intent","shape-rendering","stop-color","stop-opacity","strikethrough-position","strikethrough-thickness","stroke-dasharray","stroke-dashoffset","stroke-linecap","stroke-linejoin","stroke-miterlimit","stroke-opacity","stroke-width","text-anchor","text-decoration","text-rendering","underline-position","underline-thickness","unicode-bidi","unicode-range","units-per-em","v-alphabetic","v-hanging","v-ideographic","v-mathematical","vector-effect","vert-adv-y","vert-origin-x","vert-origin-y","word-spacing","writing-mode","xmlns:xlink","x-height"].forEach(function(n){var a=n.replace(gr,xo);Cn[a]=new Fn(a,Ze,!1,n,null,!1,!1)}),["xlink:actuate","xlink:arcrole","xlink:role","xlink:show","xlink:title","xlink:type"].forEach(function(n){var a=n.replace(gr,xo);Cn[a]=new Fn(a,Ze,!1,n,"http://www.w3.org/1999/xlink",!1,!1)}),["xml:base","xml:lang","xml:space"].forEach(function(n){var a=n.replace(gr,xo);Cn[a]=new Fn(a,Ze,!1,n,"http://www.w3.org/XML/1998/namespace",!1,!1)}),["tabIndex","crossOrigin"].forEach(function(n){Cn[n]=new Fn(n,Ze,!1,n.toLowerCase(),null,!1,!1)});var Qo="xlinkHref";Cn[Qo]=new Fn("xlinkHref",Ze,!1,"xlink:href","http://www.w3.org/1999/xlink",!0,!1),["src","href","action","formAction"].forEach(function(n){Cn[n]=new Fn(n,Ze,!1,n.toLowerCase(),null,!0,!0)});var Jo=/^[\u0000-\u001F ]*j[\r\n\t]*a[\r\n\t]*v[\r\n\t]*a[\r\n\t]*s[\r\n\t]*c[\r\n\t]*r[\r\n\t]*i[\r\n\t]*p[\r\n\t]*t[\r\n\t]*\:/i,wo=!1;function Co(n){!wo&&Jo.test(n)&&(wo=!0,u("A future version of React will block javascript: URLs as a security precaution. Use event handlers instead if you can. If you need to generate unsafe HTML try using dangerouslySetInnerHTML instead. React was passed %s.",JSON.stringify(n)))}function cr(n,a,o,f){if(f.mustUseProperty){var p=f.propertyName;return n[p]}else{ar(o,a),f.sanitizeURL&&Co(""+o);var y=f.attributeName,w=null;if(f.type===ke){if(n.hasAttribute(y)){var T=n.getAttribute(y);return T===""?!0:Tn(a,o,f,!1)?T:T===""+o?o:T}}else if(n.hasAttribute(y)){if(Tn(a,o,f,!1))return n.getAttribute(y);if(f.type===Tt)return o;w=n.getAttribute(y)}return Tn(a,o,f,!1)?w===null?o:w:w===""+o?o:w}}function Oa(n,a,o,f){{if(!wn(a))return;if(!n.hasAttribute(a))return o===void 0?void 0:null;var p=n.getAttribute(a);return ar(o,a),p===""+o?o:p}}function Di(n,a,o,f){var p=ca(a);if(!Dn(a,p,f)){if(Tn(a,o,p,f)&&(o=null),f||p===null){if(wn(a)){var y=a;o===null?n.removeAttribute(y):(ar(o,a),n.setAttribute(y,""+o))}return}var w=p.mustUseProperty;if(w){var T=p.propertyName;if(o===null){var O=p.type;n[T]=O===Tt?!1:""}else n[T]=o;return}var F=p.attributeName,H=p.attributeNamespace;if(o===null)n.removeAttribute(F);else{var ee=p.type,J;ee===Tt||ee===ke&&o===!0?J="":(ar(o,F),J=""+o,p.sanitizeURL&&Co(J.toString())),H?n.setAttributeNS(H,F,J):n.setAttribute(F,J)}}}var ka=Symbol.for("react.element"),q=Symbol.for("react.portal"),Be=Symbol.for("react.fragment"),qe=Symbol.for("react.strict_mode"),bt=Symbol.for("react.profiler"),nn=Symbol.for("react.provider"),gn=Symbol.for("react.context"),tt=Symbol.for("react.forward_ref"),$t=Symbol.for("react.suspense"),zn=Symbol.for("react.suspense_list"),In=Symbol.for("react.memo"),an=Symbol.for("react.lazy"),Lr=Symbol.for("react.scope"),ci=Symbol.for("react.debug_trace_mode"),_o=Symbol.for("react.offscreen"),da=Symbol.for("react.legacy_hidden"),Qi=Symbol.for("react.cache"),yu=Symbol.for("react.tracing_marker"),fi=Symbol.iterator,zs="@@iterator";function Pi(n){if(n===null||typeof n!="object")return null;var a=fi&&n[fi]||n[zs];return typeof a=="function"?a:null}var Zt=Object.assign,Mi=0,pa,Eo,es,To,ts,ns,No;function Ro(){}Ro.__reactDisabledLog=!0;function rs(){{if(Mi===0){pa=console.log,Eo=console.info,es=console.warn,To=console.error,ts=console.group,ns=console.groupCollapsed,No=console.groupEnd;var n={configurable:!0,enumerable:!0,value:Ro,writable:!0};Object.defineProperties(console,{info:n,log:n,warn:n,error:n,group:n,groupCollapsed:n,groupEnd:n})}Mi++}}function Bs(){{if(Mi--,Mi===0){var n={configurable:!0,enumerable:!0,writable:!0};Object.defineProperties(console,{log:Zt({},n,{value:pa}),info:Zt({},n,{value:Eo}),warn:Zt({},n,{value:es}),error:Zt({},n,{value:To}),group:Zt({},n,{value:ts}),groupCollapsed:Zt({},n,{value:ns}),groupEnd:Zt({},n,{value:No})})}Mi<0&&u("disabledDepth fell below zero. This is a bug in React. Please file an issue.")}}var Us=r.ReactCurrentDispatcher,di;function Dr(n,a,o){{if(di===void 0)try{throw Error()}catch(p){var f=p.stack.trim().match(/\n( *(at )?)/);di=f&&f[1]||""}return` +`+di+n}}var Da=!1,ha;{var Pa=typeof WeakMap=="function"?WeakMap:Map;ha=new Pa}function as(n,a){if(!n||Da)return"";{var o=ha.get(n);if(o!==void 0)return o}var f;Da=!0;var p=Error.prepareStackTrace;Error.prepareStackTrace=void 0;var y;y=Us.current,Us.current=null,rs();try{if(a){var w=function(){throw Error()};if(Object.defineProperty(w.prototype,"props",{set:function(){throw Error()}}),typeof Reflect=="object"&&Reflect.construct){try{Reflect.construct(w,[])}catch(me){f=me}Reflect.construct(n,[],w)}else{try{w.call()}catch(me){f=me}n.call(w.prototype)}}else{try{throw Error()}catch(me){f=me}n()}}catch(me){if(me&&f&&typeof me.stack=="string"){for(var T=me.stack.split(` +`),O=f.stack.split(` +`),F=T.length-1,H=O.length-1;F>=1&&H>=0&&T[F]!==O[H];)H--;for(;F>=1&&H>=0;F--,H--)if(T[F]!==O[H]){if(F!==1||H!==1)do if(F--,H--,H<0||T[F]!==O[H]){var ee=` +`+T[F].replace(" at new "," at ");return n.displayName&&ee.includes("")&&(ee=ee.replace("",n.displayName)),typeof n=="function"&&ha.set(n,ee),ee}while(F>=1&&H>=0);break}}}finally{Da=!1,Us.current=y,Bs(),Error.prepareStackTrace=p}var J=n?n.displayName||n.name:"",he=J?Dr(J):"";return typeof n=="function"&&ha.set(n,he),he}function Ao(n,a,o){return as(n,!0)}function is(n,a,o){return as(n,!1)}function Nl(n){var a=n.prototype;return!!(a&&a.isReactComponent)}function ma(n,a,o){if(n==null)return"";if(typeof n=="function")return as(n,Nl(n));if(typeof n=="string")return Dr(n);switch(n){case $t:return Dr("Suspense");case zn:return Dr("SuspenseList")}if(typeof n=="object")switch(n.$$typeof){case tt:return is(n.render);case In:return ma(n.type,a,o);case an:{var f=n,p=f._payload,y=f._init;try{return ma(y(p),a,o)}catch{}}}return""}function Oo(n){switch(n._debugOwner&&n._debugOwner.type,n._debugSource,n.tag){case _:return Dr(n.type);case de:return Dr("Lazy");case V:return Dr("Suspense");case be:return Dr("SuspenseList");case m:case b:case X:return is(n.type);case z:return is(n.type.render);case v:return Ao(n.type);default:return""}}function Ma(n){try{var a="",o=n;do a+=Oo(o),o=o.return;while(o);return a}catch(f){return` +Error generating stack: `+f.message+` +`+f.stack}}function pi(n,a,o){var f=n.displayName;if(f)return f;var p=a.displayName||a.name||"";return p!==""?o+"("+p+")":o}function js(n){return n.displayName||"Context"}function on(n){if(n==null)return null;if(typeof n.tag=="number"&&u("Received an unexpected object in getComponentNameFromType(). This is likely a bug in React. Please file an issue."),typeof n=="function")return n.displayName||n.name||null;if(typeof n=="string")return n;switch(n){case Be:return"Fragment";case q:return"Portal";case bt:return"Profiler";case qe:return"StrictMode";case $t:return"Suspense";case zn:return"SuspenseList"}if(typeof n=="object")switch(n.$$typeof){case gn:var a=n;return js(a)+".Consumer";case nn:var o=n;return js(o._context)+".Provider";case tt:return pi(n,n.render,"ForwardRef");case In:var f=n.displayName||null;return f!==null?f:on(n.type)||"Memo";case an:{var p=n,y=p._payload,w=p._init;try{return on(w(y))}catch{return null}}}return null}function hi(n,a,o){var f=a.displayName||a.name||"";return n.displayName||(f!==""?o+"("+f+")":o)}function ko(n){return n.displayName||"Context"}function It(n){var a=n.tag,o=n.type;switch(a){case Re:return"Cache";case j:var f=o;return ko(f)+".Consumer";case M:var p=o;return ko(p._context)+".Provider";case se:return"DehydratedFragment";case z:return hi(o,o.render,"ForwardRef");case A:return"Fragment";case _:return o;case C:return"Portal";case x:return"Root";case N:return"Text";case de:return on(o);case I:return o===qe?"StrictMode":"Mode";case ie:return"Offscreen";case U:return"Profiler";case Xe:return"Scope";case V:return"Suspense";case be:return"SuspenseList";case Ce:return"TracingMarker";case v:case m:case ve:case b:case K:case X:if(typeof o=="function")return o.displayName||o.name||null;if(typeof o=="string")return o;break}return null}var $s=r.ReactDebugCurrentFrame,yr=null,Ii=!1;function ja(){{if(yr===null)return null;var n=yr._debugOwner;if(n!==null&&typeof n<"u")return It(n)}return null}function Ji(){return yr===null?"":Ma(yr)}function Cr(){$s.getCurrentStack=null,yr=null,Ii=!1}function er(n){$s.getCurrentStack=n===null?null:Ji,yr=n,Ii=!1}function Do(){return yr}function Vr(n){Ii=n}function ur(n){return""+n}function ea(n){switch(typeof n){case"boolean":case"number":case"string":case"undefined":return n;case"object":return hn(n),n;default:return""}}var bu={button:!0,checkbox:!0,image:!0,hidden:!0,radio:!0,reset:!0,submit:!0};function eo(n,a){bu[a.type]||a.onChange||a.onInput||a.readOnly||a.disabled||a.value==null||u("You provided a `value` prop to a form field without an `onChange` handler. This will render a read-only field. If the field should be mutable use `defaultValue`. Otherwise, set either `onChange` or `readOnly`."),a.onChange||a.readOnly||a.disabled||a.checked==null||u("You provided a `checked` prop to a form field without an `onChange` handler. This will render a read-only field. If the field should be mutable use `defaultChecked`. Otherwise, set either `onChange` or `readOnly`.")}function mi(n){var a=n.type,o=n.nodeName;return o&&o.toLowerCase()==="input"&&(a==="checkbox"||a==="radio")}function Vs(n){return n._valueTracker}function pn(n){n._valueTracker=null}function Pn(n){var a="";return n&&(mi(n)?a=n.checked?"true":"false":a=n.value),a}function Rl(n){var a=mi(n)?"checked":"value",o=Object.getOwnPropertyDescriptor(n.constructor.prototype,a);hn(n[a]);var f=""+n[a];if(!(n.hasOwnProperty(a)||typeof o>"u"||typeof o.get!="function"||typeof o.set!="function")){var p=o.get,y=o.set;Object.defineProperty(n,a,{configurable:!0,get:function(){return p.call(this)},set:function(T){hn(T),f=""+T,y.call(this,T)}}),Object.defineProperty(n,a,{enumerable:o.enumerable});var w={getValue:function(){return f},setValue:function(T){hn(T),f=""+T},stopTracking:function(){pn(n),delete n[a]}};return w}}function Ia(n){Vs(n)||(n._valueTracker=Rl(n))}function W(n){if(!n)return!1;var a=Vs(n);if(!a)return!0;var o=a.getValue(),f=Pn(n);return f!==o?(a.setValue(f),!0):!1}function Q(n){if(n=n||(typeof document<"u"?document:void 0),typeof n>"u")return null;try{return n.activeElement||n.body}catch{return n.body}}var fe=!1,at=!1,sn=!1,Mn=!1;function Vt(n){var a=n.type==="checkbox"||n.type==="radio";return a?n.checked!=null:n.value!=null}function S(n,a){var o=n,f=a.checked,p=Zt({},a,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:f??o._wrapperState.initialChecked});return p}function R(n,a){eo("input",a),a.checked!==void 0&&a.defaultChecked!==void 0&&!at&&(u("%s contains an input of type %s with both checked and defaultChecked props. Input elements must be either controlled or uncontrolled (specify either the checked prop, or the defaultChecked prop, but not both). Decide between using a controlled or uncontrolled input element and remove one of these props. More info: https://reactjs.org/link/controlled-components",ja()||"A component",a.type),at=!0),a.value!==void 0&&a.defaultValue!==void 0&&!fe&&(u("%s contains an input of type %s with both value and defaultValue props. Input elements must be either controlled or uncontrolled (specify either the value prop, or the defaultValue prop, but not both). Decide between using a controlled or uncontrolled input element and remove one of these props. More info: https://reactjs.org/link/controlled-components",ja()||"A component",a.type),fe=!0);var o=n,f=a.defaultValue==null?"":a.defaultValue;o._wrapperState={initialChecked:a.checked!=null?a.checked:a.defaultChecked,initialValue:ea(a.value!=null?a.value:f),controlled:Vt(a)}}function $(n,a){var o=n,f=a.checked;f!=null&&Di(o,"checked",f,!1)}function G(n,a){var o=n;{var f=Vt(a);!o._wrapperState.controlled&&f&&!Mn&&(u("A component is changing an uncontrolled input to be controlled. This is likely caused by the value changing from undefined to a defined value, which should not happen. Decide between using a controlled or uncontrolled input element for the lifetime of the component. More info: https://reactjs.org/link/controlled-components"),Mn=!0),o._wrapperState.controlled&&!f&&!sn&&(u("A component is changing a controlled input to be uncontrolled. This is likely caused by the value changing from a defined to undefined, which should not happen. Decide between using a controlled or uncontrolled input element for the lifetime of the component. More info: https://reactjs.org/link/controlled-components"),sn=!0)}$(n,a);var p=ea(a.value),y=a.type;if(p!=null)y==="number"?(p===0&&o.value===""||o.value!=p)&&(o.value=ur(p)):o.value!==ur(p)&&(o.value=ur(p));else if(y==="submit"||y==="reset"){o.removeAttribute("value");return}a.hasOwnProperty("value")?$e(o,a.type,p):a.hasOwnProperty("defaultValue")&&$e(o,a.type,ea(a.defaultValue)),a.checked==null&&a.defaultChecked!=null&&(o.defaultChecked=!!a.defaultChecked)}function te(n,a,o){var f=n;if(a.hasOwnProperty("value")||a.hasOwnProperty("defaultValue")){var p=a.type,y=p==="submit"||p==="reset";if(y&&(a.value===void 0||a.value===null))return;var w=ur(f._wrapperState.initialValue);o||w!==f.value&&(f.value=w),f.defaultValue=w}var T=f.name;T!==""&&(f.name=""),f.defaultChecked=!f.defaultChecked,f.defaultChecked=!!f._wrapperState.initialChecked,T!==""&&(f.name=T)}function Fe(n,a){var o=n;G(o,a),xe(o,a)}function xe(n,a){var o=a.name;if(a.type==="radio"&&o!=null){for(var f=n;f.parentNode;)f=f.parentNode;ar(o,"name");for(var p=f.querySelectorAll("input[name="+JSON.stringify(""+o)+'][type="radio"]'),y=0;y.")))}):a.dangerouslySetInnerHTML!=null&&(Nt||(Nt=!0,u("Pass a `value` prop if you set dangerouslyInnerHTML so React knows which value should be selected.")))),a.selected!=null&&!it&&(u("Use the `defaultValue` or `value` props on must be a scalar value if `multiple` is false.%s",o,va())}}}}function Nn(n,a,o,f){var p=n.options;if(a){for(var y=o,w={},T=0;T.");var f=Zt({},a,{value:void 0,defaultValue:void 0,children:ur(o._wrapperState.initialValue)});return f}function fv(n,a){var o=n;eo("textarea",a),a.value!==void 0&&a.defaultValue!==void 0&&!Ub&&(u("%s contains a textarea with both value and defaultValue props. Textarea elements must be either controlled or uncontrolled (specify either the value prop, or the defaultValue prop, but not both). Decide between using a controlled or uncontrolled textarea and remove one of these props. More info: https://reactjs.org/link/controlled-components",ja()||"A component"),Ub=!0);var f=a.value;if(f==null){var p=a.children,y=a.defaultValue;if(p!=null){u("Use the `defaultValue` or `value` props instead of setting children on