From 84c10346fb777c827df58b264a726572225a45c6 Mon Sep 17 00:00:00 2001 From: James Reynolds <magnsuviri@me.com> Date: Wed, 31 Aug 2022 03:29:37 -0600 Subject: [PATCH 1/7] check if torch.backends has mps before calling it --- ldm/dream/devices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/dream/devices.py b/ldm/dream/devices.py index 9008f7871c..240754dd36 100644 --- a/ldm/dream/devices.py +++ b/ldm/dream/devices.py @@ -4,7 +4,7 @@ def choose_torch_device() -> str: '''Convenience routine for guessing which GPU device to run model on''' if torch.cuda.is_available(): return 'cuda' - if torch.backends.mps.is_available(): + if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): return 'mps' return 'cpu' From f4a19af04f134669da298ce9ecb61bcd01107a0d Mon Sep 17 00:00:00 2001 From: gabrielrotbart <gabe@rotbart.net> Date: Fri, 2 Sep 2022 14:55:24 +0300 Subject: [PATCH 2/7] fix scope being set to autocast even for m1 --- ldm/simplet2i.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 49a9fd38c8..b46ecbae43 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -280,7 +280,10 @@ class T2I: ), 'can only work with strength in [0.0, 1.0]' width, height, _ = self._resolution_check(width, height, log=True) - scope = autocast if self.precision == 'autocast' else nullcontext + if self.precision == 'autocast' and torch.cuda.is_available(): + scope = autocast + else: + scope = nullcontext if sampler_name and (sampler_name != self.sampler_name): self.sampler_name = sampler_name From 1306457b2713f0ada754dea3c8f0e98081ec06a0 Mon Sep 17 00:00:00 2001 From: James Reynolds <magnsuviri@me.com> Date: Fri, 2 Sep 2022 08:17:19 -0600 Subject: [PATCH 3/7] README-Mac update --- README-Mac-MPS.md | 119 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/README-Mac-MPS.md b/README-Mac-MPS.md index e7f6594132..2ddd029a95 100644 --- a/README-Mac-MPS.md +++ b/README-Mac-MPS.md @@ -1,22 +1,30 @@ -# Apple Silicon Mac Users +# macOS Instructions -Several people have gotten Stable Diffusion to work on Apple Silicon -Macs using Anaconda, miniforge, etc. I've gathered up most of their instructions and -put them in this fork (and readme). Things have moved really fast and so these -instructions change often. Hopefully things will settle down a little. +Requirements -There's several places where people are discussing Apple -MPS functionality: [the original CompVis -issue](https://github.com/CompVis/stable-diffusion/issues/25), and generally on -[lstein's fork](https://github.com/lstein/stable-diffusion/). +- macOS 12.3 Monterey or later +- Python +- Patience +- Apple Silicon* -You have to have macOS 12.3 Monterey or later. Anything earlier than that won't work. +*I haven't tested any of this on Intel Macs but I have read that one person got +it to work, so Apple Silicon might not be requried. -Tested on a 2022 Macbook M2 Air with 10-core GPU and 24 GB unified memory. +Things have moved really fast and so these instructions change often and are +often out-of-date. One of the problems is that there are so many different ways to +run this. -How to: +We are trying to build a testing setup so that when we make changes it doesn't +always break. + +How to (this hasn't been 100% tested yet): + +First [download the model](https://huggingface.co/CompVis/stable-diffusion). ``` +brew install --cask miniconda +brew install Cmake protobuf rust + git clone https://github.com/lstein/stable-diffusion.git cd stable-diffusion @@ -24,6 +32,7 @@ mkdir -p models/ldm/stable-diffusion-v1/ PATH_TO_CKPT="$HOME/Documents/stable-diffusion-v-1-4-original" # or wherever yours is. ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt +export PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml conda activate ldm @@ -31,7 +40,17 @@ python scripts/preload_models.py python scripts/dream.py --full_precision # half-precision requires autocast and won't work ``` -After you follow all the instructions and run dream.py you might get several errors. Here's the errors I've seen and found solutions for. +The original scripts should work as well. + +``` +python scripts/orig_scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms +``` + +Note, `export PIP_EXISTS_ACTION=w` is a precaution to fix `conda env create -f environment-mac.yaml` +never finishing in some situations. So it isn't required but wont hurt. + +After you follow all the instructions and run dream.py you might get several +errors. Here's the errors I've seen and found solutions for. ### Is it slow? @@ -52,27 +71,37 @@ One debugging step is to update to the latest version of PyTorch nightly. conda install pytorch torchvision torchaudio -c pytorch-nightly -Or you can clean everything up. +If `conda env create -f environment-mac.yaml` takes forever run this. + + git clean -f + +And run this. conda clean --yes --all -Or you can reset Anaconda. +Or you could reset Anaconda. conda update --force-reinstall -y -n base -c defaults conda -### "No module named cv2" (or some other module) +### "No module named cv2", torch, 'ldm', 'transformers', 'taming', etc. -Did you remember to `conda activate ldm`? If your terminal prompt +There are several causes of these errors. + +First, did you remember to `conda activate ldm`? If your terminal prompt begins with "(ldm)" then you activated it. If it begins with "(base)" or something else you haven't. -If it says you're missing taming you need to rebuild your virtual +Second, you might've run `./scripts/preload_models.py` or `./scripts/dream.py` +instead of `python ./scripts/preload_models.py` or `python ./scripts/dream.py`. +The cause of this error is long so it's below. + +Third, if it says you're missing taming you need to rebuild your virtual environment. conda env remove -n ldm conda env create -f environment-mac.yaml -If you have activated the ldm virtual environment and tried rebuilding +Fourth, If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't and you'll just need to manually install it. Make sure you activate the virtual environment so it installs there instead of @@ -83,6 +112,56 @@ globally. You might also need to install Rust (I mention this again below). +### How many snakes are living in your computer? + +Here's the reason why you have to specify which python to use. +There are several versions of python on macOS and the computer is +picking the wrong one. More specifically, preload_models.py and dream.py says to +find the first `python3` in the path environment variable. You can see which one +it is picking with `which python3`. These are the mostly likely paths you'll see. + + % which python3 + /usr/bin/python3 + +The above path is part of the OS. However, that path is a stub that asks you if +you want to install Xcode. If you have Xcode installed already, +/usr/bin/python3 will execute /Library/Developer/CommandLineTools/usr/bin/python3 or +/Applications/Xcode.app/Contents/Developer/usr/bin/python3 (depending on which +Xcode you've selected with `xcode-select`). + + % which python3 + /opt/homebrew/bin/python3 + +If you installed python3 with Homebrew and you've modified your path to search +for Homebrew binaries before system ones, you'll see the above path. + + % which python + /opt/anaconda3/bin/python + +If you drop the "3" you get an entirely different python. Note: starting in +macOS 12.3, /usr/bin/python no longer exists (it was python 2 anyway). + +If you have Anaconda installed, this is what you'll see. There is a +/opt/anaconda3/bin/python3 also. + + (ldm) % which python + /Users/name/miniforge3/envs/ldm/bin/python + +This is what you'll see if you have miniforge and you've correctly activated +the ldm environment. This is the goal. + +It's all a mess and you should know [how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) +if you want to fix it. Here's a brief hint of all the ways you can modify it +(don't really have the time to explain it all here). + +- ~/.zshrc +- ~/.bash_profile +- ~/.bashrc +- /etc/paths.d +- /etc/path + +Which one you use will depend on what you have installed except putting a file +in /etc/paths.d is what I prefer to do. ### Debugging? @@ -139,7 +218,7 @@ the environment variable `CONDA_SUBDIR=osx-arm64`, like so: This error happens with Anaconda on Macs when the Intel-only `mkl` is pulled in by a dependency. [nomkl](https://stackoverflow.com/questions/66224879/what-is-the-nomkl-python-package-used-for) is a metapackage designed to prevent this, by making it impossible to install -`mkl`, but if your environment is already broken it may not work. +`mkl`, but if your environment is already broken it may not work. Do *not* use `os.environ['KMP_DUPLICATE_LIB_OK']='True'` or equivalents as this masks the underlying issue of using Intel packages. From 0b582a40d0b017220edd3b5610e47f9139609376 Mon Sep 17 00:00:00 2001 From: Lincoln Stein <lincoln.stein@gmail.com> Date: Fri, 2 Sep 2022 10:17:51 -0400 Subject: [PATCH 4/7] add developer's guidance for refactoring this change --- ldm/simplet2i.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index b46ecbae43..10720a7483 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -280,6 +280,10 @@ class T2I: ), 'can only work with strength in [0.0, 1.0]' width, height, _ = self._resolution_check(width, height, log=True) + + # TODO: - Check if this is still necessary to run on M1 devices. + # - Move code into ldm.dream.devices to live alongside other + # special-hardware casing code. if self.precision == 'autocast' and torch.cuda.is_available(): scope = autocast else: From 1701c2ea943aafe9b051fc2f461d88806b113aa8 Mon Sep 17 00:00:00 2001 From: James Reynolds <magnsuviri@me.com> Date: Fri, 2 Sep 2022 08:17:19 -0600 Subject: [PATCH 5/7] README-Mac update --- README-Mac-MPS.md | 119 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/README-Mac-MPS.md b/README-Mac-MPS.md index e7f6594132..2ddd029a95 100644 --- a/README-Mac-MPS.md +++ b/README-Mac-MPS.md @@ -1,22 +1,30 @@ -# Apple Silicon Mac Users +# macOS Instructions -Several people have gotten Stable Diffusion to work on Apple Silicon -Macs using Anaconda, miniforge, etc. I've gathered up most of their instructions and -put them in this fork (and readme). Things have moved really fast and so these -instructions change often. Hopefully things will settle down a little. +Requirements -There's several places where people are discussing Apple -MPS functionality: [the original CompVis -issue](https://github.com/CompVis/stable-diffusion/issues/25), and generally on -[lstein's fork](https://github.com/lstein/stable-diffusion/). +- macOS 12.3 Monterey or later +- Python +- Patience +- Apple Silicon* -You have to have macOS 12.3 Monterey or later. Anything earlier than that won't work. +*I haven't tested any of this on Intel Macs but I have read that one person got +it to work, so Apple Silicon might not be requried. -Tested on a 2022 Macbook M2 Air with 10-core GPU and 24 GB unified memory. +Things have moved really fast and so these instructions change often and are +often out-of-date. One of the problems is that there are so many different ways to +run this. -How to: +We are trying to build a testing setup so that when we make changes it doesn't +always break. + +How to (this hasn't been 100% tested yet): + +First [download the model](https://huggingface.co/CompVis/stable-diffusion). ``` +brew install --cask miniconda +brew install Cmake protobuf rust + git clone https://github.com/lstein/stable-diffusion.git cd stable-diffusion @@ -24,6 +32,7 @@ mkdir -p models/ldm/stable-diffusion-v1/ PATH_TO_CKPT="$HOME/Documents/stable-diffusion-v-1-4-original" # or wherever yours is. ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt +export PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml conda activate ldm @@ -31,7 +40,17 @@ python scripts/preload_models.py python scripts/dream.py --full_precision # half-precision requires autocast and won't work ``` -After you follow all the instructions and run dream.py you might get several errors. Here's the errors I've seen and found solutions for. +The original scripts should work as well. + +``` +python scripts/orig_scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms +``` + +Note, `export PIP_EXISTS_ACTION=w` is a precaution to fix `conda env create -f environment-mac.yaml` +never finishing in some situations. So it isn't required but wont hurt. + +After you follow all the instructions and run dream.py you might get several +errors. Here's the errors I've seen and found solutions for. ### Is it slow? @@ -52,27 +71,37 @@ One debugging step is to update to the latest version of PyTorch nightly. conda install pytorch torchvision torchaudio -c pytorch-nightly -Or you can clean everything up. +If `conda env create -f environment-mac.yaml` takes forever run this. + + git clean -f + +And run this. conda clean --yes --all -Or you can reset Anaconda. +Or you could reset Anaconda. conda update --force-reinstall -y -n base -c defaults conda -### "No module named cv2" (or some other module) +### "No module named cv2", torch, 'ldm', 'transformers', 'taming', etc. -Did you remember to `conda activate ldm`? If your terminal prompt +There are several causes of these errors. + +First, did you remember to `conda activate ldm`? If your terminal prompt begins with "(ldm)" then you activated it. If it begins with "(base)" or something else you haven't. -If it says you're missing taming you need to rebuild your virtual +Second, you might've run `./scripts/preload_models.py` or `./scripts/dream.py` +instead of `python ./scripts/preload_models.py` or `python ./scripts/dream.py`. +The cause of this error is long so it's below. + +Third, if it says you're missing taming you need to rebuild your virtual environment. conda env remove -n ldm conda env create -f environment-mac.yaml -If you have activated the ldm virtual environment and tried rebuilding +Fourth, If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't and you'll just need to manually install it. Make sure you activate the virtual environment so it installs there instead of @@ -83,6 +112,56 @@ globally. You might also need to install Rust (I mention this again below). +### How many snakes are living in your computer? + +Here's the reason why you have to specify which python to use. +There are several versions of python on macOS and the computer is +picking the wrong one. More specifically, preload_models.py and dream.py says to +find the first `python3` in the path environment variable. You can see which one +it is picking with `which python3`. These are the mostly likely paths you'll see. + + % which python3 + /usr/bin/python3 + +The above path is part of the OS. However, that path is a stub that asks you if +you want to install Xcode. If you have Xcode installed already, +/usr/bin/python3 will execute /Library/Developer/CommandLineTools/usr/bin/python3 or +/Applications/Xcode.app/Contents/Developer/usr/bin/python3 (depending on which +Xcode you've selected with `xcode-select`). + + % which python3 + /opt/homebrew/bin/python3 + +If you installed python3 with Homebrew and you've modified your path to search +for Homebrew binaries before system ones, you'll see the above path. + + % which python + /opt/anaconda3/bin/python + +If you drop the "3" you get an entirely different python. Note: starting in +macOS 12.3, /usr/bin/python no longer exists (it was python 2 anyway). + +If you have Anaconda installed, this is what you'll see. There is a +/opt/anaconda3/bin/python3 also. + + (ldm) % which python + /Users/name/miniforge3/envs/ldm/bin/python + +This is what you'll see if you have miniforge and you've correctly activated +the ldm environment. This is the goal. + +It's all a mess and you should know [how to modify the path environment variable](https://support.apple.com/guide/terminal/use-environment-variables-apd382cc5fa-4f58-4449-b20a-41c53c006f8f/mac) +if you want to fix it. Here's a brief hint of all the ways you can modify it +(don't really have the time to explain it all here). + +- ~/.zshrc +- ~/.bash_profile +- ~/.bashrc +- /etc/paths.d +- /etc/path + +Which one you use will depend on what you have installed except putting a file +in /etc/paths.d is what I prefer to do. ### Debugging? @@ -139,7 +218,7 @@ the environment variable `CONDA_SUBDIR=osx-arm64`, like so: This error happens with Anaconda on Macs when the Intel-only `mkl` is pulled in by a dependency. [nomkl](https://stackoverflow.com/questions/66224879/what-is-the-nomkl-python-package-used-for) is a metapackage designed to prevent this, by making it impossible to install -`mkl`, but if your environment is already broken it may not work. +`mkl`, but if your environment is already broken it may not work. Do *not* use `os.environ['KMP_DUPLICATE_LIB_OK']='True'` or equivalents as this masks the underlying issue of using Intel packages. From 6ef7c1ad4e6cc662b7c36f8363e037e45828e6cd Mon Sep 17 00:00:00 2001 From: James Reynolds <magnsuviri@me.com> Date: Fri, 2 Sep 2022 08:29:28 -0600 Subject: [PATCH 6/7] Added psychedelicious' changes --- README-Mac-MPS.md | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/README-Mac-MPS.md b/README-Mac-MPS.md index 2ddd029a95..5476a66ed0 100644 --- a/README-Mac-MPS.md +++ b/README-Mac-MPS.md @@ -19,24 +19,45 @@ always break. How to (this hasn't been 100% tested yet): -First [download the model](https://huggingface.co/CompVis/stable-diffusion). +First get the weights checkpoint download started - it's big: + +Sign up at https://huggingface.co +Accept the terms and click Access Repository: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original +Download sd-v1-4.ckpt (4.27 GB) and note where you have saved it (probably the Downloads folder) +While that is downloading, open Terminal and run the following commands one at a time. ``` -brew install --cask miniconda -brew install Cmake protobuf rust +# install brew (and Xcode command line tools): +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# install python 3, git, cmake, protobuf: +brew install cmake protobuf rust + +# install miniconda (M1 arm64 version): +curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o Miniconda3-latest-MacOSX-arm64.sh +/bin/bash Miniconda3-latest-MacOSX-arm64.sh + +# clone the repo git clone https://github.com/lstein/stable-diffusion.git cd stable-diffusion +# +# wait until the checkpoint file has downloaded, then proceed +# + +# create symlink to checkpoint mkdir -p models/ldm/stable-diffusion-v1/ -PATH_TO_CKPT="$HOME/Documents/stable-diffusion-v-1-4-original" # or wherever yours is. +PATH_TO_CKPT="$HOME/Downloads" # or wherever you saved sd-v1-4.ckpt ln -s "$PATH_TO_CKPT/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt -export PIP_EXISTS_ACTION=w -CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml +# install packages +PIP_EXISTS_ACTION=w CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac.yaml conda activate ldm +# only need to do this once python scripts/preload_models.py + +# run SD! python scripts/dream.py --full_precision # half-precision requires autocast and won't work ``` From ad6cf6f2f7c1311edfc471474c60bbd192492241 Mon Sep 17 00:00:00 2001 From: Simon Vans-Colina <simon@vans-colina.com> Date: Fri, 2 Sep 2022 23:45:46 +0800 Subject: [PATCH 7/7] Update readme to make it clearer for Windows users --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 38cb46f681..78d473be61 100644 --- a/README.md +++ b/README.md @@ -329,8 +329,10 @@ and introducing a new vocabulary to the fixed model. To train, prepare a folder that contains images sized at 512x512 and execute the following: + +WINDOWS: As the default backend is not available on Windows, if you're using that platform, set the environment variable `PL_TORCH_DISTRIBUTED_BACKEND=gloo` + ``` -# As the default backend is not available on Windows, if you're using that platform, execute SET PL_TORCH_DISTRIBUTED_BACKEND=gloo (ldm) ~/stable-diffusion$ python3 ./main.py --base ./configs/stable-diffusion/v1-finetune.yaml \ -t \ --actual_resume ./models/ldm/stable-diffusion-v1/model.ckpt \