From 044e1ec2a815dd50c56c45fc3ea9eda5930bdd27 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 09:55:48 -0500 Subject: [PATCH 1/8] Add requirements file for linux container on Apple silicon --- requirements-linux-arm64.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 requirements-linux-arm64.txt diff --git a/requirements-linux-arm64.txt b/requirements-linux-arm64.txt new file mode 100644 index 0000000000..24ad623d84 --- /dev/null +++ b/requirements-linux-arm64.txt @@ -0,0 +1,24 @@ +albumentations==0.4.3 +einops==0.3.0 +huggingface-hub==0.8.1 +imageio==2.9.0 +imageio-ffmpeg==0.4.2 +kornia==0.6.0 +numpy==1.23.1 +--pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu +omegaconf==2.1.1 +opencv-python==4.6.0.66 +pillow==9.2.0 +pudb==2019.2 +torch==1.12.1 +torchvision==0.12.0 +pytorch-lightning==1.4.2 +streamlit==1.12.0 +test-tube>=0.7.5 +torch-fidelity==0.3.0 +torchmetrics==0.6.0 +transformers==4.19.2 +-e git+https://github.com/openai/CLIP.git@main#egg=clip +-e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers +git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion +-e . \ No newline at end of file From 4a0354c60443b5f9083516207fc995b5b79d6b03 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:28:15 -0500 Subject: [PATCH 2/8] Add README for Mac-Docker --- README-Mac-Docker.md | 234 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 README-Mac-Docker.md diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md new file mode 100644 index 0000000000..dc51944a37 --- /dev/null +++ b/README-Mac-Docker.md @@ -0,0 +1,234 @@ +Table of Contents +================= + +Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. + +* [Setup](#setup) + * [Directly on Apple silicon](#directly-on-apple-silicon) + * [Prerequisites](#prerequisites) + * [Set up](#set-up) + * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Prerequisites](#prerequisites-1) + * [Launch and set up a container](#launch-and-set-up-a-container) + * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) +* [Usage](#usage) + * [Startup](#startup) + * [Text to Image](#text-to-image) + * [Image to Image](#image-to-image) + * [Web Interface](#web-interface) + * [Notes](#notes) + +# Setup + +## Directly on Apple silicon + +For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). + +### Prerequisites +Install the latest versions of macOS, [Homebrew](https://brew.sh/), [Python](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#python), and [Git](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#git). + +```Shell +brew install cmake protobuf rust +brew install --cask miniconda +conda init zsh && source ~/.zshrc # or bash and .bashrc +``` + +### Set up +```Shell +GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +mkdir -p models/ldm/stable-diffusion-v1/ +``` + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: + +```Shell +PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt +ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 +conda env create -f environment-mac.yaml && conda activate ldm +``` + +You can verify you're in the virtual environment by looking at which executable you're getting: +```Shell +type python3 +``` + +Only need to do this once: +```Shell +python3 scripts/preload_models.py +``` + +## On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --name dummy --mount source=my-vol,target=/data alpine +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Launch and set up a container +Start a container for Stable Diffusion +```Shell +docker run -it \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +debian +# or arm64v8/debian +``` + +You're now on the container. Set it up: +```Shell +apt update && apt upgrade -y && apt install -y \ +git \ +pip3 \ +python3 \ +wget + +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + +# you won't need to close and reopen your terminal after this because we'll source our .rc file +cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc +# update conda +conda update -y -n base -c defaults conda + +cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion + +# When path exists, pip3 will (w)ipe. +# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. +PIP_EXISTS_ACTION=w +CONDA_SUBDIR=osx-arm64 + +# Create the environment +# conda env create -f environment.yaml && conda activate ldm +conda create -y --name ldm && conda activate ldm +pip3 install -r requirements-linux-arm64.txt + +python3 scripts/preload_models.py + +mkdir -p models/ldm/stable-diffusion-v1 \ +&& chown root:root /data/sd-v1-4.ckpt \ +&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt +``` + +## [Optional] Face Restoration and Upscaling +```Shell +cd .. # by default expected in a sibling directory +git clone https://github.com/TencentARC/GFPGAN.git +cd GFPGAN + +pip3 install basicsr # used for training and inference +pip3 install facexlib # face detection and face restoration helper +pip3 install -r requirements.txt + +python3 setup.py develop +pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling +# pre-trained model needed for face restoration +wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models + +cd .. +cd stable-diffusion +python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +``` + +# Usage + +## Startup +With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: +Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. + +By default the images are saved in ```outputs/img-samples/```. +If you're on a docker container set the output dir to the Docker volume. +```Shell +# If on Macbook +python3 scripts/dream.py --full_precision +# If on Linux container +python3 scripts/dream.py --full_precision -o /data +``` + +You'll get the script's prompt. You can see available options or quit. +```Shell +dream> -h +dream> q +``` + +## Text to Image +For quick (and rough) results test with 5 steps (default 50), 1 sample image. +Increase steps to 100 or more for good (but slower) results. +The prompt can be in quotes or not. +``` +dream> The hulk fighting with sheldon cooper -s5 -n1 +dream> "woman closeup highly detailed" -s 150 +# Reuse previous seed and apply face restoration (if you installed GFPGAN) +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +``` +TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. + +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +```Shell +# On your host Macbook (you can use the name of any container that mounted the volume) +docker cp dummy:/data/ ~/Pictures +``` + +## Image to Image +You can also do text-guided image-to-image translation. For example, turning a sketch into a detailed drawing. +Strength is a value between 0.0 and 1.0, that controls the amount of noise that is added to the input image. Values that approach 1.0 allow for lots of variations but will also produce images that are not semantically consistent with the input. 0.0 preserves image exactly, 1.0 replaces it completely. +Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Otherwise you'll get ```Error: product of dimension sizes > 2**31'```. If you still get the error [try a different size](https://support.apple.com/guide/preview/resize-rotate-or-flip-an-image-prvw2015/mac#:~:text=image's%20file%20size-,In%20the%20Preview%20app%20on%20your%20Mac%2C%20open%20the%20file,is%20shown%20at%20the%20bottom.) like 512x256. + +If you're on a docker container, copy your input image into the Docker volume +```Shell +docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +``` + +Try it out generating an image (or 4). +```Shell +# If you're on your Macbook +dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +# If you're on a Linux container on your Macbook +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +``` + +## Web Interface +You can use the script with a graphical web interface +```Shell +python3 scripts/dream.py --full_precision --web +``` +and point your browser to http://127.0.0.1:9090 + +## Notes + +Some text you can add at the end of the prompt to make it very pretty: +```Shell +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +``` + +The original scripts should work as well. +```Shell +python3 scripts/orig_scripts/txt2img.py --help +python3 scripts/orig_scripts/txt2img.py --ddim_steps 100 --n_iter 1 --n_samples 1 --plms --prompt "new born baby kitten. Hyper Detail, Octane Rendering, Unreal Engine, V-Ray" +python3 scripts/orig_scripts/txt2img.py --ddim_steps 5 --n_iter 1 --n_samples 1 --plms --prompt "ocean" # or --klms +``` + From 6d2084e0303c88e8ebe031468ca870dc69f1c4a6 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 15:46:14 -0500 Subject: [PATCH 3/8] Clean up instructioins --- README-Mac-Docker.md | 51 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index dc51944a37..ae51416e6f 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -1,16 +1,16 @@ + Table of Contents ================= -Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. - -* [Setup](#setup) - * [Directly on Apple silicon](#directly-on-apple-silicon) +* [Installation](#installation) + * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) * [Prerequisites](#prerequisites) - * [Set up](#set-up) - * [On a Linux container with Docker for Apple silicon](#on-a-linux-container-with-docker-for-apple-silicon) + * [Setup](#setup) + * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites-1) - * [Launch and set up a container](#launch-and-set-up-a-container) + * [Setup](#setup-1) * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) + * [Setup](#setup-2) * [Usage](#usage) * [Startup](#startup) * [Text to Image](#text-to-image) @@ -18,10 +18,9 @@ Tested on **MacBook Air M2** with **Docker Desktop for Mac with Apple Chip**. * [Web Interface](#web-interface) * [Notes](#notes) -# Setup +# Installation -## Directly on Apple silicon - +## Option 1 - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -33,7 +32,8 @@ brew install --cask miniconda conda init zsh && source ~/.zshrc # or bash and .bashrc ``` -### Set up +### Setup +Set it to the fork you want to use. ```Shell GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git @@ -65,7 +65,7 @@ Only need to do this once: python3 scripts/preload_models.py ``` -## On a Linux container with Docker for Apple silicon +## Option 2 - On a Linux container with Docker for Apple silicon You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. ### Prerequisites @@ -87,7 +87,7 @@ cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` -### Launch and set up a container +### Setup Start a container for Stable Diffusion ```Shell docker run -it \ @@ -99,16 +99,16 @@ debian # or arm64v8/debian ``` -You're now on the container. Set it up: +You're now on the container. Set the fork you want to use and set up the container: ```Shell +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" + apt update && apt upgrade -y && apt install -y \ git \ pip3 \ python3 \ wget -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc @@ -135,13 +135,15 @@ mkdir -p models/ldm/stable-diffusion-v1 \ ``` ## [Optional] Face Restoration and Upscaling -```Shell -cd .. # by default expected in a sibling directory -git clone https://github.com/TencentARC/GFPGAN.git -cd GFPGAN +Whether you're directly on macOS or a Linux container. -pip3 install basicsr # used for training and inference -pip3 install facexlib # face detection and face restoration helper +### Setup +```Shell +# by default expected in a sibling directory to stable-diffusion +cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN + +# basicsr: used for training and inference. facexlib: face detection / face restoration helper. +pip3 install basicsr facexlib pip3 install -r requirements.txt python3 setup.py develop @@ -149,8 +151,7 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up # pre-trained model needed for face restoration wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models -cd .. -cd stable-diffusion +cd ../stable-diffusion python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. ``` @@ -222,7 +223,7 @@ and point your browser to http://127.0.0.1:9090 Some text you can add at the end of the prompt to make it very pretty: ```Shell -cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed concept art painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida +cinematic photo, highly detailed, cinematic lighting, ultra-detailed, ultrarealistic, photorealism, Octane Rendering, cyberpunk lights, Hyper Detail, 8K, HD, Unreal Engine, V-Ray, full hd, cyberpunk, abstract, 3d octane render + 4k UHD + immense detail + dramatic lighting + well lit + black, purple, blue, pink, cerulean, teal, metallic colours, + fine details, ultra photoreal, photographic, concept art, cinematic composition, rule of thirds, mysterious, eerie, photorealism, breathtaking detailed, painting art deco pattern, by hsiao, ron cheng, john james audubon, bizarre compositions, exquisite detail, extremely moody lighting, painted by greg rutkowski makoto shinkai takashi takeuchi studio ghibli, akihiko yoshida ``` The original scripts should work as well. From 4185afea5c299b146123a49abde1d6d2fac91ba4 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Fri, 9 Sep 2022 23:15:09 -0500 Subject: [PATCH 4/8] Update documentation --- README-Mac-Docker.md | 85 +++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index ae51416e6f..503fe7f442 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -18,6 +18,10 @@ Table of Contents * [Web Interface](#web-interface) * [Notes](#notes) + +Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. +You'll need to create an account but it's quick and free. + # Installation ## Option 1 - Directly on Apple silicon @@ -33,8 +37,9 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ``` ### Setup -Set it to the fork you want to use. + ```Shell +# Set the fork you want to use. GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git git clone $GITHUB_STABLE_DIFFUSION @@ -42,8 +47,6 @@ cd stable-diffusion mkdir -p models/ldm/stable-diffusion-v1/ ``` -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB). You'll need to create an account but it's quick and free. Then set up the environment: - ```Shell PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt @@ -70,10 +73,7 @@ You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https: ### Prerequisites [Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Adjust the CPUs and Memory to the largest amount available to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. -You'll need to create an account but it's quick and free. +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. Create a Docker volume for the downloaded model file ``` @@ -82,38 +82,39 @@ docker volume create my-vol Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. ```Shell -docker create --name dummy --mount source=my-vol,target=/data alpine +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine cd ~/Downloads # or wherever you saved sd-v1-4.ckpt docker cp sd-v1-4.ckpt dummy:/data ``` ### Setup -Start a container for Stable Diffusion +Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. ```Shell docker run -it \ --platform linux/arm64 \ --name stable-diffusion \ --hostname stable-diffusion \ --mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 80:9090 \ debian # or arm64v8/debian ``` -You're now on the container. Set the fork you want to use and set up the container: +You're now on the container. ```Shell -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" - -apt update && apt upgrade -y && apt install -y \ +# Set the fork you want to use +GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ +&& apt update && apt upgrade -y \ +&& apt install -y \ git \ -pip3 \ +pip \ python3 \ wget # you won't need to close and reopen your terminal after this because we'll source our .rc file cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc -# update conda -conda update -y -n base -c defaults conda cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion @@ -122,11 +123,11 @@ cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 -# Create the environment -# conda env create -f environment.yaml && conda activate ldm -conda create -y --name ldm && conda activate ldm -pip3 install -r requirements-linux-arm64.txt +# Create the environment, activate it, install requirements. +conda create -y --name ldm && conda activate ldm \ +&& pip3 install -r requirements-linux-arm64.txt +# Only need to do this once (ok twice if you decide to add face restoration and upscaling): python3 scripts/preload_models.py mkdir -p models/ldm/stable-diffusion-v1 \ @@ -139,6 +140,9 @@ Whether you're directly on macOS or a Linux container. ### Setup ```Shell +# If you're on a Linux container +apt install -y libgl1-mesa-glx libglib2.0-0 + # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN @@ -152,9 +156,11 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -python3 scripts/preload_models.py # if not, it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. +python3 scripts/preload_models.py ``` + # Usage ## Startup @@ -162,12 +168,12 @@ With the Conda environment activated (```conda activate ldm```), run the interac Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume. +If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision # If on Linux container -python3 scripts/dream.py --full_precision -o /data +python3 scripts/dream.py --full_precision -o /data ``` You'll get the script's prompt. You can see available options or quit. @@ -177,21 +183,25 @@ dream> q ``` ## Text to Image -For quick (and rough) results test with 5 steps (default 50), 1 sample image. -Increase steps to 100 or more for good (but slower) results. +For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. -``` +```Shell dream> The hulk fighting with sheldon cooper -s5 -n1 dream> "woman closeup highly detailed" -s 150 # Reuse previous seed and apply face restoration (if you installed GFPGAN) -dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.8 +dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 +# TODO: example for upscaling. ``` -TODO: example for upscaling. The -U option currently [doesn't work](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it easily wherever you want. +If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +You can download it from the Docker Desktop app, Volumes, my-vol, data. +Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) -docker cp dummy:/data/ ~/Pictures +docker cp dummy:/data/000001.928403745.png /Users//Pictures ``` ## Image to Image @@ -201,23 +211,26 @@ Make sure your input image size dimensions are multiples of 64 e.g. 512x512. Oth If you're on a docker container, copy your input image into the Docker volume ```Shell -docker cp ~/Pictures/sketch-mountains-input.jpg dummy:/data/ +docker cp /Users//Pictures/sketch-mountains-input.jpg dummy:/data/ ``` -Try it out generating an image (or 4). +Try it out generating an image (or 4). +The ```dream``` script needs absolute paths to find the image so don't use ```~```. ```Shell # If you're on your Macbook -dream> "A fantasy landscape, trending on artstation" -I ~/Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 +dream> "A fantasy landscape, trending on artstation" -I /Users//Pictures/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n4 # If you're on a Linux container on your Macbook -dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.8 --steps 100 -n1 +dream> "A fantasy landscape, trending on artstation" -I /data/sketch-mountains-input.jpg --strength 0.75 --steps 100 -n1 ``` ## Web Interface -You can use the script with a graphical web interface +You can use the ```dream``` script with a graphical web interface. Start the web server with: ```Shell python3 scripts/dream.py --full_precision --web ``` -and point your browser to http://127.0.0.1:9090 +If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090 + +Press Control-C at the command line to stop the web server. ## Notes From 97c0c4bfe817d98e6031030450e7fac0b0bb3af7 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:38:31 -0500 Subject: [PATCH 5/8] Initial version of Dockerfile and entreypoint --- Dockerfile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ entrypoint.sh | 8 ++++++++ 2 files changed, 58 insertions(+) create mode 100644 Dockerfile create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..59382247fb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +FROM arm64v8/debian +MAINTAINER Armando C. Santisbon + +ARG gsd +ENV GITHUB_STABLE_DIFFUSION $gsd + +ARG sdreq="requirements-linux-arm64.txt" +ENV SD_REQ $sdreq + +WORKDIR / +COPY entrypoint.sh anaconda.sh . +SHELL ["/bin/bash", "-c"] + +RUN apt update && apt upgrade -y \ + && apt install -y \ + git \ + pip \ + python3 \ + wget \ + # install Anaconda or Miniconda + && chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc \ + && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion \ + # When path exists, pip3 will (w)ipe. + && PIP_EXISTS_ACTION="w" \ + # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. + && CONDA_SUBDIR="osx-arm64" \ + # Create the environment, activate it, install requirements. + && conda create -y --name ldm && conda activate ldm \ + && pip3 install -r $SD_REQ \ + + # Only need to do this once (we'll do it after we add face restoration and upscaling): + # && python3 scripts/preload_models.py \ + + && mkdir models/ldm/stable-diffusion-v1 \ + # [Optional] Face Restoration and Upscaling + && apt install -y libgl1-mesa-glx libglib2.0-0 \ + # by default expected in a sibling directory to stable-diffusion + && cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN \ + && pip3 install basicsr facexlib \ + && pip3 install -r requirements.txt \ + && python3 setup.py develop \ + # to enhance the background (non-face) regions and do upscaling + && pip3 install realesrgan \ + # pre-trained model needed for face restoration + && wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models \ + && cd ../stable-diffusion \ + # if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. + && python3 scripts/preload_models.py + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000000..26967ab8ae --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt + +if [ $# -eq 0 ]; then + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data +else + python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" +fi From f9239af7dc6f1c57bc51ad26ca8321cc989822e3 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 12:42:05 -0500 Subject: [PATCH 6/8] Ignore Anaconda/Miniconda installer used during Docker build --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fd75e65a48..bbb6fc20ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ outputs/ models/ldm/stable-diffusion-v1/model.ckpt +# ignore the Anaconda/Miniconda installer used while building Docker image +anaconda.sh + # ignore a directory which serves as a place for initial images inputs/ From 2cf8de92344bb63ac5e7a3487a65f86b5ccbd725 Mon Sep 17 00:00:00 2001 From: "Armando C. Santisbon" Date: Sat, 10 Sep 2022 13:56:18 -0500 Subject: [PATCH 7/8] Add entrypoint and update documentation --- README-Mac-Docker.md | 186 +++++++++++++++++++------------------------ entrypoint.sh | 6 +- 2 files changed, 87 insertions(+), 105 deletions(-) diff --git a/README-Mac-Docker.md b/README-Mac-Docker.md index 503fe7f442..eda202152c 100644 --- a/README-Mac-Docker.md +++ b/README-Mac-Docker.md @@ -2,29 +2,84 @@ Table of Contents ================= -* [Installation](#installation) - * [Option 1 - Directly on Apple silicon](#option-1---directly-on-apple-silicon) +* [Step 1 - Get the Model](#step-1---get-the-model) +* [Step 2 - Installation](#step-2---installation) + * [Option A - On a Linux container with Docker for Apple silicon](#option-a---on-a-linux-container-with-docker-for-apple-silicon) * [Prerequisites](#prerequisites) * [Setup](#setup) - * [Option 2 - On a Linux container with Docker for Apple silicon](#option-2---on-a-linux-container-with-docker-for-apple-silicon) + * [Option B - Directly on Apple silicon](#option-b---directly-on-apple-silicon) * [Prerequisites](#prerequisites-1) * [Setup](#setup-1) - * [[Optional] Face Restoration and Upscaling](#optional-face-restoration-and-upscaling) - * [Setup](#setup-2) -* [Usage](#usage) +* [Step 3 - Usage (time to have fun)](#step-3---usage-time-to-have-fun) * [Startup](#startup) * [Text to Image](#text-to-image) * [Image to Image](#image-to-image) * [Web Interface](#web-interface) * [Notes](#notes) - +# Step 1 - Get the Model Go to [Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), and click "Access repository" to Download ```sd-v1-4.ckpt``` (~4 GB) to ```~/Downloads```. You'll need to create an account but it's quick and free. -# Installation +# Step 2 - Installation -## Option 1 - Directly on Apple silicon +## Option A - On a Linux container with Docker for Apple silicon +You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. + +### Prerequisites +[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) +On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. + +Create a Docker volume for the downloaded model file +``` +docker volume create my-vol +``` + +Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. +```Shell +docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine + +# Copy the model file to the Docker volume. We'll need it at run time. +cd ~/Downloads # or wherever you saved sd-v1-4.ckpt +docker cp sd-v1-4.ckpt dummy:/data +``` + +### Setup +```Shell +# Set the fork you want to use. +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" + +git clone $GITHUB_STABLE_DIFFUSION +cd stable-diffusion +chmod +x entrypoint.sh +# download the Miniconda installer. We'll need it at build time. +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ +``` + +Build the Docker image. Give it any tag ```-t``` that you want. +Tip: Make sure your shell session has the env variable set (above) with ```echo $GITHUB_STABLE_DIFFUSION```. +```Shell +docker build -t santisbon/stable-diffusion \ +--build-arg gsd=$GITHUB_STABLE_DIFFUSION \ +--build-arg sdreq="requirements-linux-arm64.txt" \ +. +``` + +Run a container using your built image e.g. +```Shell +docker run -it \ +--rm \ +--platform linux/arm64 \ +--name stable-diffusion \ +--hostname stable-diffusion \ +--mount source=my-vol,target=/data \ +--expose 9090 \ +--publish 9090:9090 \ +santisbon/stable-diffusion +``` +Tip: Make sure you've created the Docker volume (above) + +## Option B - Directly on Apple silicon For Mac M1/M2. Read more about [Metal Performance Shaders (MPS) framework](https://developer.apple.com/documentation/metalperformanceshaders). ### Prerequisites @@ -40,7 +95,7 @@ conda init zsh && source ~/.zshrc # or bash and .bashrc ```Shell # Set the fork you want to use. -GITHUB_STABLE_DIFFUSION=https://github.com/santisbon/stable-diffusion.git +GITHUB_STABLE_DIFFUSION="https://github.com/santisbon/stable-diffusion.git" git clone $GITHUB_STABLE_DIFFUSION cd stable-diffusion @@ -53,8 +108,8 @@ ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt # When path exists, pip3 will (w)ipe. # restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 +PIP_EXISTS_ACTION="w" +CONDA_SUBDIR="osx-arm64" conda env create -f environment-mac.yaml && conda activate ldm ``` @@ -63,92 +118,15 @@ You can verify you're in the virtual environment by looking at which executable type python3 ``` -Only need to do this once: +Face Restoration and Upscaling ```Shell -python3 scripts/preload_models.py -``` - -## Option 2 - On a Linux container with Docker for Apple silicon -You [can't access the Macbook M1/M2 GPU cores from the Docker containers](https://github.com/pytorch/pytorch/issues/81224) so performance is reduced but for development purposes it's fine. - -### Prerequisites -[Install Docker](https://gist.github.com/santisbon/2165fd1c9aaa1f7974f424535d3756f7#install-2) -On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the CPUs and Memory to avoid this [Issue](https://github.com/lstein/stable-diffusion/issues/342). You may need to increase Swap and Disk image size too. - -Create a Docker volume for the downloaded model file -``` -docker volume create my-vol -``` - -Populate the volume using a lightweight Linux container. You just need to create the container with the mountpoint; no need to run it. -```Shell -docker create --platform linux/arm64 --name dummy --mount source=my-vol,target=/data alpine # or arm64v8/alpine -cd ~/Downloads # or wherever you saved sd-v1-4.ckpt -docker cp sd-v1-4.ckpt dummy:/data -``` - -### Setup -Start a container for Stable Diffusion. The container's 9090 port is mapped to the host's 80. That way you'll be able to use the Web interface from your Mac. -```Shell -docker run -it \ ---platform linux/arm64 \ ---name stable-diffusion \ ---hostname stable-diffusion \ ---mount source=my-vol,target=/data \ ---expose 9090 \ ---publish 80:9090 \ -debian -# or arm64v8/debian -``` - -You're now on the container. -```Shell -# Set the fork you want to use -GITHUB_STABLE_DIFFUSION="-b docker-apple-silicon https://github.com/santisbon/stable-diffusion.git" \ -&& apt update && apt upgrade -y \ -&& apt install -y \ -git \ -pip \ -python3 \ -wget - -# you won't need to close and reopen your terminal after this because we'll source our .rc file -cd /data && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh \ -&& chmod +x anaconda.sh && bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash && source ~/.bashrc - -cd / && git clone $GITHUB_STABLE_DIFFUSION && cd stable-diffusion - -# When path exists, pip3 will (w)ipe. -# restrict the Conda environment to only use ARM packages. M1/M2 is ARM-based. You could also conda install nomkl. -PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 - -# Create the environment, activate it, install requirements. -conda create -y --name ldm && conda activate ldm \ -&& pip3 install -r requirements-linux-arm64.txt - -# Only need to do this once (ok twice if you decide to add face restoration and upscaling): -python3 scripts/preload_models.py - -mkdir -p models/ldm/stable-diffusion-v1 \ -&& chown root:root /data/sd-v1-4.ckpt \ -&& ln -sf /data/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt -``` - -## [Optional] Face Restoration and Upscaling -Whether you're directly on macOS or a Linux container. - -### Setup -```Shell -# If you're on a Linux container -apt install -y libgl1-mesa-glx libglib2.0-0 # by default expected in a sibling directory to stable-diffusion cd .. && git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN # basicsr: used for training and inference. facexlib: face detection / face restoration helper. -pip3 install basicsr facexlib -pip3 install -r requirements.txt +pip3 install basicsr facexlib \ +&& pip3 install -r requirements.txt python3 setup.py develop pip3 install realesrgan # to enhance the background (non-face) regions and do upscaling @@ -156,24 +134,26 @@ pip3 install realesrgan # to enhance the background (non-face) regions and do up wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P experiments/pretrained_models cd ../stable-diffusion -# if we don't preload models it will download model files from the Internet the first time you run dream.py with GFPGAN and Real-ESRGAN turned on. -python3 scripts/preload_models.py ``` +Only need to do this once. If we don't preload models it will download model files from the Internet when you run ```dream.py```. +```Shell +python3 scripts/preload_models.py +``` -# Usage +# Step 3 - Usage (time to have fun) ## Startup +If you're on a Linux container the ```dream``` script is automatically started and the output dir set to the Docker volume you created earlier. + +If you're directly on macOS follow these startup instructions. With the Conda environment activated (```conda activate ldm```), run the interactive interface that combines the functionality of the original scripts txt2img and img2img: Use the more accurate but VRAM-intensive full precision math because half-precision requires autocast and won't work. -By default the images are saved in ```outputs/img-samples/```. -If you're on a docker container set the output dir to the Docker volume you created. ```Shell # If on Macbook python3 scripts/dream.py --full_precision -# If on Linux container -python3 scripts/dream.py --full_precision -o /data +# By default the images are saved in outputs/img-samples/. ``` You'll get the script's prompt. You can see available options or quit. @@ -183,7 +163,7 @@ dream> q ``` ## Text to Image -For quick (but very rough) results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. +For quick (but bad) image results test with 5 steps (default 50) and 1 sample image. This will let you know that everything is set up correctly. Then increase steps to 100 or more for good (but slower) results. The prompt can be in quotes or not. ```Shell @@ -194,11 +174,11 @@ dream> "woman closeup highly detailed" --steps 150 --seed -1 -G 0.75 # TODO: example for upscaling. ``` You'll need to experiment to see if face restoration is making it better or worse for your specific prompt. -The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297) on Mac. +The -U option for upscaling has an [Issue](https://github.com/lstein/stable-diffusion/issues/297). -If you're on a container and set the output to the Docker volume (or moved it there with ```mv outputs/img-samples/ /data/```) you can copy it wherever you want. +If you're on a container the output is set to the Docker volume. You can copy it wherever you want. You can download it from the Docker Desktop app, Volumes, my-vol, data. -Or you can copy it from your terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: +Or you can copy it from your Mac terminal. Keep in mind ```docker cp``` can't expand ```*.png``` so you'll need to specify the image file name: ```Shell # On your host Macbook (you can use the name of any container that mounted the volume) docker cp dummy:/data/000001.928403745.png /Users//Pictures diff --git a/entrypoint.sh b/entrypoint.sh index 26967ab8ae..c7cc9af8d3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,8 +1,10 @@ #!/bin/bash ln -sf /data/sd-v1-4.ckpt /stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt +cd /stable-diffusion +conda activate ldm if [ $# -eq 0 ]; then - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data + python3 scripts/dream.py --full_precision -o /data else - python3 /stable-diffusion/scripts/dream.py --full_precision -o /data "$@" + python3 scripts/dream.py --full_precision -o /data "$@" fi From 529fc57f2bd22b742011823887de47e27e0716af Mon Sep 17 00:00:00 2001 From: Mihai <299015+mh-dm@users.noreply.github.com> Date: Sat, 10 Sep 2022 16:58:07 +0300 Subject: [PATCH 8/8] ~7% speedup (1.57 to 1.69it/s) from switch to += in ldm.modules.attention. (#482) Tested on 8GB eGPU nvidia setup so YMMV. 512x512 output, max VRAM stays same. --- ldm/modules/attention.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 817e9bcdc4..24aef29279 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -235,9 +235,9 @@ class BasicTransformerBlock(nn.Module): def _forward(self, x, context=None): x = x.contiguous() if x.device.type == 'mps' else x - x = self.attn1(self.norm1(x)) + x - x = self.attn2(self.norm2(x), context=context) + x - x = self.ff(self.norm3(x)) + x + x += self.attn1(self.norm1(x)) + x += self.attn2(self.norm2(x), context=context) + x += self.ff(self.norm3(x)) return x