From 3e48b9ff85fc9cce5010fa733969a70341a7cf66 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 22 Oct 2022 23:02:50 -0400 Subject: [PATCH 01/12] cut over from karras to model noise schedule for higher steps The k_samplers come with a "karras" noise schedule which performs very well at low step counts but becomes noisy at higher ones. This commit introduces a threshold (currently 30 steps) at which the k samplers will switch over from using karras to the older model noise schedule. --- ldm/models/diffusion/ksampler.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ldm/models/diffusion/ksampler.py b/ldm/models/diffusion/ksampler.py index ac0615b30c..677e051be7 100644 --- a/ldm/models/diffusion/ksampler.py +++ b/ldm/models/diffusion/ksampler.py @@ -12,6 +12,10 @@ from ldm.modules.diffusionmodules.util import ( extract_into_tensor, ) +# at this threshold, the scheduler will stop using the Karras +# noise schedule and start using the model's schedule +STEP_THRESHOLD = 30 + def cfg_apply_threshold(result, threshold = 0.0, scale = 0.7): if threshold <= 0.0: return result @@ -98,8 +102,13 @@ class KSampler(Sampler): rho=7., device=self.device, ) - self.sigmas = self.model_sigmas - #self.sigmas = self.karras_sigmas + + if ddim_num_steps >= STEP_THRESHOLD: + print(f'>> number of steps ({ddim_num_steps}) >= {STEP_THRESHOLD}: using model sigmas') + self.sigmas = self.model_sigmas + else: + print(f'>> number of steps ({ddim_num_steps}) < {STEP_THRESHOLD}: using karras sigmas') + self.sigmas = self.karras_sigmas # ALERT: We are completely overriding the sample() method in the base class, which # means that inpainting will not work. To get this to work we need to be able to From e9f690bf9d0bc794c229ffd9e7c3d75a54daf47c Mon Sep 17 00:00:00 2001 From: Taylor Kems Date: Thu, 27 Oct 2022 07:19:26 -0500 Subject: [PATCH 02/12] Update IMG2IMG.md --- docs/features/IMG2IMG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/features/IMG2IMG.md b/docs/features/IMG2IMG.md index a540ff5cc9..99c72d2a73 100644 --- a/docs/features/IMG2IMG.md +++ b/docs/features/IMG2IMG.md @@ -121,8 +121,6 @@ Both of the outputs look kind of like what I was thinking of. With the strength If you want to try this out yourself, all of these are using a seed of `1592514025` with a width/height of `384`, step count `10`, the default sampler (`k_lms`), and the single-word prompt `"fire"`: -If you want to try this out yourself, all of these are using a seed of `1592514025` with a width/height of `384`, step count `10`, the default sampler (`k_lms`), and the single-word prompt `fire`: - ```commandline invoke> "fire" -s10 -W384 -H384 -S1592514025 -I /tmp/fire-drawing.png --strength 0.7 ``` From b2a3c5cbe8ddc810b5117062898a2fce1a159083 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:01:10 +0200 Subject: [PATCH 03/12] update Dockerfile --- docker-build/Dockerfile | 109 +++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/docker-build/Dockerfile b/docker-build/Dockerfile index f3d6834c93..2f0c892730 100644 --- a/docker-build/Dockerfile +++ b/docker-build/Dockerfile @@ -1,57 +1,74 @@ -FROM debian +FROM ubuntu AS get_miniconda -ARG gsd -ENV GITHUB_STABLE_DIFFUSION $gsd +SHELL ["/bin/bash", "-c"] -ARG rsd -ENV REQS $rsd +# install wget +RUN apt-get update \ + && apt-get install -y \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -ARG cs -ENV CONDA_SUBDIR $cs +# download and install miniconda +ARG conda_version=py39_4.12.0-Linux-x86_64 +ARG conda_prefix=/opt/conda +RUN wget --progress=dot:giga -O /miniconda.sh \ + https://repo.anaconda.com/miniconda/Miniconda3-${conda_version}.sh \ + && bash /miniconda.sh -b -p ${conda_prefix} \ + && rm -f /miniconda.sh -ENV PIP_EXISTS_ACTION="w" +FROM ubuntu AS invokeai -# TODO: Optimize image size +# use bash +SHELL [ "/bin/bash", "-c" ] -SHELL ["/bin/bash", "-c"] +# clean bashrc +RUN echo "" > ~/.bashrc -WORKDIR / -RUN apt update && apt upgrade -y \ - && apt install -y \ - git \ - libgl1-mesa-glx \ - libglib2.0-0 \ - pip \ - python3 \ - && git clone $GITHUB_STABLE_DIFFUSION +# Install necesarry packages +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + gcc \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 \ + pip \ + python3 \ + python3-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -# Install Anaconda or Miniconda -COPY anaconda.sh . -RUN bash anaconda.sh -b -u -p /anaconda && /anaconda/bin/conda init bash +# clone repository and create symlinks +ARG invokeai_git=https://github.com/invoke-ai/InvokeAI.git +ARG project_name=invokeai +RUN git clone ${invokeai_git} /${project_name} \ + && mkdir /${project_name}/models/ldm/stable-diffusion-v1 \ + && ln -s /data/models/sd-v1-4.ckpt /${project_name}/models/ldm/stable-diffusion-v1/model.ckpt \ + && ln -s /data/outputs/ /${project_name}/outputs -# SD -WORKDIR /stable-diffusion -RUN source ~/.bashrc \ - && conda create -y --name ldm && conda activate ldm \ - && conda config --env --set subdir $CONDA_SUBDIR \ - && pip3 install -r $REQS \ - && pip3 install basicsr facexlib realesrgan \ - && mkdir models/ldm/stable-diffusion-v1 \ - && ln -s "/data/sd-v1-4.ckpt" models/ldm/stable-diffusion-v1/model.ckpt +# set workdir +WORKDIR /${project_name} -# Face restoreation -# by default expected in a sibling directory to stable-diffusion -WORKDIR / -RUN git clone https://github.com/TencentARC/GFPGAN.git +# install conda env and preload models +ARG conda_prefix=/opt/conda +ARG conda_env_file=environment.yml +COPY --from=get_miniconda ${conda_prefix} ${conda_prefix} +RUN source ${conda_prefix}/etc/profile.d/conda.sh \ + && conda init bash \ + && source ~/.bashrc \ + && conda env create \ + --name ${project_name} \ + --file ${conda_env_file} \ + && rm -Rf ~/.cache \ + && conda clean -afy \ + && echo "conda activate ${project_name}" >> ~/.bashrc \ + && ln -s /data/models/GFPGANv1.4.pth ./src/gfpgan/experiments/pretrained_models/GFPGANv1.4.pth \ + && conda activate ${project_name} \ + && python scripts/preload_models.py -WORKDIR /GFPGAN -RUN pip3 install -r requirements.txt \ - && python3 setup.py develop \ - && ln -s "/data/GFPGANv1.4.pth" experiments/pretrained_models/GFPGANv1.4.pth - -WORKDIR /stable-diffusion -RUN python3 scripts/preload_models.py - -WORKDIR / -COPY entrypoint.sh . -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +# Copy entrypoint and set env +ENV CONDA_PREFIX=${conda_prefix} +ENV PROJECT_NAME=${project_name} +COPY docker-build/entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] From 44731f8a3794f3b83e5ea68dedc0a33e76000d98 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:02:07 +0200 Subject: [PATCH 04/12] add .dockerignore to repo-root since transfering the 2GB context would not be rly productive --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..8a0ebc5069 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!environment*.yml +!docker-build From 624fe4794bf5e72905b202d25668141f5204c813 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:02:58 +0200 Subject: [PATCH 05/12] add build script which also creates volume it needs a token from huggingface to be able to download the checkpoint --- docker-build/build.sh | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 docker-build/build.sh diff --git a/docker-build/build.sh b/docker-build/build.sh new file mode 100755 index 0000000000..fea311ae82 --- /dev/null +++ b/docker-build/build.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -e +# IMPORTANT: You need to have a token on huggingface.co to be able to download the checkpoint!!! +# configure values by using env when executing build.sh +# f.e. env ARCH=aarch64 GITHUB_INVOKE_AI=https://github.com/yourname/yourfork.git ./build.sh + +source ./docker-build/env.sh || echo "please run from repository root" || exit 1 + +invokeai_conda_version=${INVOKEAI_CONDA_VERSION:-py39_4.12.0-${platform/\//-}} +invokeai_conda_prefix=${INVOKEAI_CONDA_PREFIX:-\/opt\/conda} +invokeai_conda_env_file=${INVOKEAI_CONDA_ENV_FILE:-environment.yml} +invokeai_git=${INVOKEAI_GIT:-https://github.com/invoke-ai/InvokeAI.git} +huggingface_token=${HUGGINGFACE_TOKEN?} + +# print the settings +echo "You are using these values:" +echo -e "project_name:\t\t ${project_name}" +echo -e "volumename:\t\t ${volumename}" +echo -e "arch:\t\t\t ${arch}" +echo -e "platform:\t\t ${platform}" +echo -e "invokeai_conda_version:\t ${invokeai_conda_version}" +echo -e "invokeai_conda_prefix:\t ${invokeai_conda_prefix}" +echo -e "invokeai_conda_env_file: ${invokeai_conda_env_file}" +echo -e "invokeai_git:\t\t ${invokeai_git}" +echo -e "invokeai_tag:\t\t ${invokeai_tag}\n" + +_runAlpine() { + docker run \ + --rm \ + --interactive \ + --tty \ + --mount source="$volumename",target=/data \ + --workdir /data \ + alpine "$@" +} + +_copyCheckpoints() { + echo "creating subfolders for models and outputs" + _runAlpine mkdir models + _runAlpine mkdir outputs + echo -n "downloading sd-v1-4.ckpt" + _runAlpine wget --header="Authorization: Bearer ${huggingface_token}" -O models/sd-v1-4.ckpt https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt + echo "done" + echo "downloading GFPGANv1.4.pth" + _runAlpine wget -O models/GFPGANv1.4.pth https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth +} + +_checkVolumeContent() { + _runAlpine ls -lhA /data/models +} + +_getModelMd5s() { + _runAlpine \ + alpine sh -c "md5sum /data/models/*" +} + +if [[ -n "$(docker volume ls -f name="${volumename}" -q)" ]]; then + echo "Volume already exists" + if [[ -z "$(_checkVolumeContent)" ]]; then + echo "looks empty, copying checkpoint" + _copyCheckpoints + fi + echo "Models in ${volumename}:" + _checkVolumeContent +else + echo -n "createing docker volume " + docker volume create "${volumename}" + _copyCheckpoints +fi + +# Build Container +docker build \ + --platform="${platform}" \ + --tag "${invokeai_tag}" \ + --build-arg project_name="${project_name}" \ + --build-arg conda_version="${invokeai_conda_version}" \ + --build-arg conda_prefix="${invokeai_conda_prefix}" \ + --build-arg conda_env_file="${invokeai_conda_env_file}" \ + --build-arg invokeai_git="${invokeai_git}" \ + --file ./docker-build/Dockerfile \ + . From 3e2cf8a25912be197a2914333ccfc621ed88f170 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:03:25 +0200 Subject: [PATCH 06/12] add env.sh with variables shared in run and build --- docker-build/env.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docker-build/env.sh diff --git a/docker-build/env.sh b/docker-build/env.sh new file mode 100644 index 0000000000..36b718d362 --- /dev/null +++ b/docker-build/env.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +project_name=${PROJECT_NAME:-invokeai} +volumename=${VOLUMENAME:-${project_name}_data} +arch=${ARCH:-x86_64} +platform=${PLATFORM:-Linux/${arch}} +invokeai_tag=${INVOKEAI_TAG:-${project_name}-${arch}} + +export project_name +export volumename +export arch +export platform +export invokeai_tag From 67a7d46a292e9525e27f9b9f1f02f4dda9b44903 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:03:48 +0200 Subject: [PATCH 07/12] add script to easily run the container --- docker-build/run.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 docker-build/run.sh diff --git a/docker-build/run.sh b/docker-build/run.sh new file mode 100755 index 0000000000..3d1a564f4c --- /dev/null +++ b/docker-build/run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +source ./docker-build/env.sh || echo "please run from repository root" || exit 1 + +docker run \ + --interactive \ + --tty \ + --rm \ + --platform "$platform" \ + --name "$project_name" \ + --hostname "$project_name" \ + --mount source="$volumename",target=/data \ + --publish 9090:9090 \ + "$invokeai_tag" ${1:+$@} From bf50a68eb56a7348dfcc869c4e7f43afc51117df Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:05:52 +0200 Subject: [PATCH 08/12] update entrypoint - when run without arguments it starts the web-interface - can also be run with your own arguments - if u do so it does not start web unless u do --- docker-build/entrypoint.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker-build/entrypoint.sh b/docker-build/entrypoint.sh index f47e6669e0..7c0ca12f88 100755 --- a/docker-build/entrypoint.sh +++ b/docker-build/entrypoint.sh @@ -1,10 +1,8 @@ #!/bin/bash +set -e -cd /stable-diffusion +source "${CONDA_PREFIX}/etc/profile.d/conda.sh" +conda activate "${PROJECT_NAME}" -if [ $# -eq 0 ]; then - python3 scripts/dream.py --full_precision -o /data - # bash -else - python3 scripts/dream.py --full_precision -o /data "$@" -fi \ No newline at end of file +python scripts/invoke.py \ + ${@:---web --host=0.0.0.0} From 99eb7e6ef2c89b48d0fe4c52f20e1c7d8c2fb0f5 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:06:37 +0200 Subject: [PATCH 09/12] add conda env for linux-aarch64 - neither environment.yml nor environment-mac.yml was working --- environment-linux-aarch64.yml | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 environment-linux-aarch64.yml diff --git a/environment-linux-aarch64.yml b/environment-linux-aarch64.yml new file mode 100644 index 0000000000..f430770c4d --- /dev/null +++ b/environment-linux-aarch64.yml @@ -0,0 +1,44 @@ +name: invokeai +channels: + - pytorch + - conda-forge +dependencies: + - python>=3.9 + - pip>=20.3 + - cudatoolkit + - pytorch + - torchvision + - numpy=1.19 + - imageio=2.9.0 + - opencv=4.6.0 + - pillow=8.* + - flask=2.1.* + - flask_cors=3.0.10 + - flask-socketio=5.3.0 + - send2trash=1.8.0 + - eventlet + - albumentations=0.4.3 + - pudb=2019.2 + - imageio-ffmpeg=0.4.2 + - pytorch-lightning=1.7.7 + - streamlit + - einops=0.3.0 + - kornia=0.6 + - torchmetrics=0.7.0 + - transformers=4.21.3 + - torch-fidelity=0.3.0 + - tokenizers>=0.11.1,!=0.11.3,<0.13 + - pip: + - omegaconf==2.1.1 + - realesrgan==0.2.5.0 + - test-tube>=0.7.5 + - pyreadline3 + - dependency_injector==4.40.0 + - -e git+https://github.com/openai/CLIP.git@main#egg=clip + - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers + - -e git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion + - -e git+https://github.com/TencentARC/GFPGAN.git#egg=gfpgan + - -e git+https://github.com/invoke-ai/clipseg.git@models-rename#egg=clipseg + - -e . +variables: + PYTORCH_ENABLE_MPS_FALLBACK: 1 From 9b15b228b82307dc83a9118d2da15e2b5bf4510c Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 00:07:06 +0200 Subject: [PATCH 10/12] add action to build the container it does not push the container but verify buildability --- .github/workflows/build-container.yml | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build-container.yml diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 0000000000..97f465a550 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,39 @@ +# Building the Image without pushing to confirm it is still buildable +# confirum functionality would unfortunately need way more resources +name: build container image +on: + push: + branches: + - 'main' + - 'development' + pull_request_target: + branches: + - 'main' + - 'development' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + file: docker-build/Dockerfile + platforms: linux/amd64 + push: false + tags: ${{ github.repository }}:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache From fb4feb380b2cce8663449da50b39243f9a0afe36 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 25 Oct 2022 02:50:04 +0200 Subject: [PATCH 11/12] update docker docs --- docs/installation/INSTALL_DOCKER.md | 128 ++++++++++------------------ 1 file changed, 45 insertions(+), 83 deletions(-) diff --git a/docs/installation/INSTALL_DOCKER.md b/docs/installation/INSTALL_DOCKER.md index eb2e2ab39f..50c3d89c81 100644 --- a/docs/installation/INSTALL_DOCKER.md +++ b/docs/installation/INSTALL_DOCKER.md @@ -36,20 +36,6 @@ another environment with NVIDIA GPUs on-premises or in the cloud. ### Prerequisites -#### Get the data files - -Go to -[Hugging Face](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original), -and click "Access repository" to Download the model file `sd-v1-4.ckpt` (~4 GB) -to `~/Downloads`. You'll need to create an account but it's quick and free. - -Also download the face restoration model. - -```Shell -cd ~/Downloads -wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth -``` - #### Install [Docker](https://github.com/santisbon/guides#docker) On the Docker Desktop app, go to Preferences, Resources, Advanced. Increase the @@ -57,86 +43,61 @@ CPUs and Memory to avoid this [Issue](https://github.com/invoke-ai/InvokeAI/issues/342). You may need to increase Swap and Disk image size too. +#### Get a Huggingface-Token + +Go to [Hugging Face](https://huggingface.co/settings/tokens), create a token and +temporary place it somewhere like a open texteditor window (but dont save it!, +only keep it open, we need it in the next step) + ### Setup Set the fork you want to use and other variables. -```Shell -TAG_STABLE_DIFFUSION="santisbon/stable-diffusion" -PLATFORM="linux/arm64" -GITHUB_STABLE_DIFFUSION="-b orig-gfpgan https://github.com/santisbon/stable-diffusion.git" -REQS_STABLE_DIFFUSION="requirements-linux-arm64.txt" -CONDA_SUBDIR="osx-arm64" +!!! tip -echo $TAG_STABLE_DIFFUSION -echo $PLATFORM -echo $GITHUB_STABLE_DIFFUSION -echo $REQS_STABLE_DIFFUSION -echo $CONDA_SUBDIR + I preffer to save my env vars + in the repository root in a `.env` (or `.envrc`) file to automatically re-apply + them when I come back. + +The build- and run- scripts contain default values for almost everything, +besides the [Hugging Face Token](https://huggingface.co/settings/tokens) you +created in the last step. + +Some Suggestions of variables you may want to change besides the Token: + +| Environment-Variable | Description | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| `HUGGINGFACE_TOKEN="hg_aewirhghlawrgkjbarug2"` | This is the only required variable, without you can't get the checkpoint | +| `ARCH=aarch64` | if you are using a ARM based CPU | +| `INVOKEAI_TAG=yourname/invokeai:latest` | the Container Repository / Tag which will be used | +| `INVOKEAI_CONDA_ENV_FILE=environment-linux-aarch64.yml` | since environment.yml wouldn't work with aarch | +| `INVOKEAI_GIT="-b branchname https://github.com/username/reponame"` | if you want to use your own fork | + +#### Build the Image + +I provided a build script, which is located in `docker-build/build.sh` but still +needs to be executed from the Repository root. + +```bash +docker-build/build.sh ``` -Create a Docker volume for the downloaded model files. +The build Script not only builds the container, but also creates the docker +volume if not existing yet, or if empty it will just download the models. When +it is done you can run the container via the run script -```Shell -docker volume create my-vol +```bash +docker-build/run.sh ``` -Copy the data files to the Docker volume using a lightweight Linux container. -We'll need the models at run time. You just need to create the container with -the mountpoint; no need to run this dummy container. +When used without arguments, the container will start the website and provide +you the link to open it. But if you want to use some other parameters you can +also do so. -```Shell -cd ~/Downloads # or wherever you saved the files +!!! warning "Deprecated" -docker create --platform $PLATFORM --name dummy --mount source=my-vol,target=/data alpine - -docker cp sd-v1-4.ckpt dummy:/data -docker cp GFPGANv1.4.pth dummy:/data -``` - -Get the repo and download the Miniconda installer (we'll need it at build time). -Replace the URL with the version matching your container OS and the architecture -it will run on. - -```Shell -cd ~ -git clone $GITHUB_STABLE_DIFFUSION - -cd stable-diffusion/docker-build -chmod +x entrypoint.sh -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O anaconda.sh && chmod +x anaconda.sh -``` - -Build the Docker image. Give it any tag `-t` that you want. -Choose the Linux container's host platform: x86-64/Intel is `amd64`. Apple -silicon is `arm64`. If deploying the container to the cloud to leverage powerful -GPU instances you'll be on amd64 hardware but if you're just trying this out -locally on Apple silicon choose arm64. -The application uses libraries that need to match the host environment so use -the appropriate requirements file. -Tip: Check that your shell session has the env variables set above. - -```Shell -docker build -t $TAG_STABLE_DIFFUSION \ ---platform $PLATFORM \ ---build-arg gsd=$GITHUB_STABLE_DIFFUSION \ ---build-arg rsd=$REQS_STABLE_DIFFUSION \ ---build-arg cs=$CONDA_SUBDIR \ -. -``` - -Run a container using your built image. -Tip: Make sure you've created and populated the Docker volume (above). - -```Shell -docker run -it \ ---rm \ ---platform $PLATFORM \ ---name stable-diffusion \ ---hostname stable-diffusion \ ---mount source=my-vol,target=/data \ -$TAG_STABLE_DIFFUSION -``` + From here on it is the rest of the previous Docker-Docs, which will still + provide usefull informations for one or the other. ## Usage (time to have fun) @@ -240,7 +201,8 @@ server with: python3 scripts/invoke.py --full_precision --web ``` -If it's running on your Mac point your Mac web browser to http://127.0.0.1:9090 +If it's running on your Mac point your Mac web browser to + Press Control-C at the command line to stop the web server. From 943808b925824d537264c2a51c3c2c908a9472e6 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 27 Oct 2022 15:50:32 -0400 Subject: [PATCH 12/12] add threshold for switchover from Karras to LDM noise schedule --- docs/features/CLI.md | 1 + ldm/generate.py | 27 ++++++++++++++++----------- ldm/invoke/args.py | 10 +++++++++- ldm/models/diffusion/ksampler.py | 11 +++++++---- ldm/models/diffusion/sampler.py | 14 +++++++++++++- 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/docs/features/CLI.md b/docs/features/CLI.md index 85524f6fa9..281f837578 100644 --- a/docs/features/CLI.md +++ b/docs/features/CLI.md @@ -153,6 +153,7 @@ Here are the invoke> command that apply to txt2img: | --cfg_scale | -C | 7.5 | How hard to try to match the prompt to the generated image; any number greater than 1.0 works, but the useful range is roughly 5.0 to 20.0 | | --seed | -S | None | Set the random seed for the next series of images. This can be used to recreate an image generated previously.| | --sampler | -A| k_lms | Sampler to use. Use -h to get list of available samplers. | +| --karras_max | | 29 | When using k_* samplers, set the maximum number of steps before shifting from using the Karras noise schedule (good for low step counts) to the LatentDiffusion noise schedule (good for high step counts) This value is sticky. [29] | | --hires_fix | | | Larger images often have duplication artefacts. This option suppresses duplicates by generating the image at low res, and then using img2img to increase the resolution | | --png_compression <0-9> | -z<0-9> | 6 | Select level of compression for output files, from 0 (no compression) to 9 (max compression) | | --grid | -g | False | Turn on grid mode to return a single image combining all the images generated by this prompt | diff --git a/ldm/generate.py b/ldm/generate.py index 135ec9ca54..18d62ca24a 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -176,6 +176,7 @@ class Generate: self.free_gpu_mem = free_gpu_mem self.size_matters = True # used to warn once about large image sizes and VRAM self.txt2mask = None + self.karras_max = None # Note that in previous versions, there was an option to pass the # device to Generate(). However the device was then ignored, so @@ -253,6 +254,7 @@ class Generate: variation_amount = 0.0, threshold = 0.0, perlin = 0.0, + karras_max = None, # these are specific to img2img and inpaint init_img = None, init_mask = None, @@ -331,7 +333,8 @@ class Generate: strength = strength or self.strength self.seed = seed self.log_tokenization = log_tokenization - self.step_callback = step_callback + self.step_callback = step_callback + self.karras_max = karras_max with_variations = [] if with_variations is None else with_variations # will instantiate the model or return it from cache @@ -376,6 +379,11 @@ class Generate: self.sampler_name = sampler_name self._set_sampler() + # bit of a hack to change the cached sampler's karras threshold to + # whatever the user asked for + if karras_max is not None and isinstance(self.sampler,KSampler): + self.sampler.adjust_settings(karras_max=karras_max) + tic = time.time() if self._has_cuda(): torch.cuda.reset_peak_memory_stats() @@ -815,26 +823,23 @@ class Generate: def _set_sampler(self): msg = f'>> Setting Sampler to {self.sampler_name}' + karras_max = self.karras_max # set in generate() call if self.sampler_name == 'plms': self.sampler = PLMSSampler(self.model, device=self.device) elif self.sampler_name == 'ddim': self.sampler = DDIMSampler(self.model, device=self.device) elif self.sampler_name == 'k_dpm_2_a': - self.sampler = KSampler( - self.model, 'dpm_2_ancestral', device=self.device - ) + self.sampler = KSampler(self.model, 'dpm_2_ancestral', device=self.device, karras_max=karras_max) elif self.sampler_name == 'k_dpm_2': - self.sampler = KSampler(self.model, 'dpm_2', device=self.device) + self.sampler = KSampler(self.model, 'dpm_2', device=self.device, karras_max=karras_max) elif self.sampler_name == 'k_euler_a': - self.sampler = KSampler( - self.model, 'euler_ancestral', device=self.device - ) + self.sampler = KSampler(self.model, 'euler_ancestral', device=self.device, karras_max=karras_max) elif self.sampler_name == 'k_euler': - self.sampler = KSampler(self.model, 'euler', device=self.device) + self.sampler = KSampler(self.model, 'euler', device=self.device, karras_max=karras_max) elif self.sampler_name == 'k_heun': - self.sampler = KSampler(self.model, 'heun', device=self.device) + self.sampler = KSampler(self.model, 'heun', device=self.device, karras_max=karras_max) elif self.sampler_name == 'k_lms': - self.sampler = KSampler(self.model, 'lms', device=self.device) + self.sampler = KSampler(self.model, 'lms', device=self.device, karras_max=karras_max) else: msg = f'>> Unsupported Sampler: {self.sampler_name}, Defaulting to plms' self.sampler = PLMSSampler(self.model, device=self.device) diff --git a/ldm/invoke/args.py b/ldm/invoke/args.py index e2302e4452..2b00b5a9ce 100644 --- a/ldm/invoke/args.py +++ b/ldm/invoke/args.py @@ -216,6 +216,8 @@ class Args(object): switches.append(f'-W {a["width"]}') switches.append(f'-H {a["height"]}') switches.append(f'-C {a["cfg_scale"]}') + if a['karras_max'] is not None: + switches.append(f'--karras_max {a["karras_max"]}') if a['perlin'] > 0: switches.append(f'--perlin {a["perlin"]}') if a['threshold'] > 0: @@ -669,7 +671,13 @@ class Args(object): default=6, choices=range(0,10), dest='png_compression', - help='level of PNG compression, from 0 (none) to 9 (maximum). Default is 6.' + help='level of PNG compression, from 0 (none) to 9 (maximum). [6]' + ) + render_group.add_argument( + '--karras_max', + type=int, + default=None, + help="control the point at which the K* samplers will shift from using the Karras noise schedule (good for low step counts) to the LatentDiffusion noise schedule (good for high step counts). Set to 0 to use LatentDiffusion for all step values, and to a high value (e.g. 1000) to use Karras for all step values. [29]." ) img2img_group.add_argument( '-I', diff --git a/ldm/models/diffusion/ksampler.py b/ldm/models/diffusion/ksampler.py index 677e051be7..1693baade5 100644 --- a/ldm/models/diffusion/ksampler.py +++ b/ldm/models/diffusion/ksampler.py @@ -14,7 +14,7 @@ from ldm.modules.diffusionmodules.util import ( # at this threshold, the scheduler will stop using the Karras # noise schedule and start using the model's schedule -STEP_THRESHOLD = 30 +STEP_THRESHOLD = 29 def cfg_apply_threshold(result, threshold = 0.0, scale = 0.7): if threshold <= 0.0: @@ -64,6 +64,9 @@ class KSampler(Sampler): self.sigmas = None self.ds = None self.s_in = None + self.karras_max = kwargs.get('karras_max',STEP_THRESHOLD) + if self.karras_max is None: + self.karras_max = STEP_THRESHOLD def forward(self, x, sigma, uncond, cond, cond_scale): x_in = torch.cat([x] * 2) @@ -103,11 +106,11 @@ class KSampler(Sampler): device=self.device, ) - if ddim_num_steps >= STEP_THRESHOLD: - print(f'>> number of steps ({ddim_num_steps}) >= {STEP_THRESHOLD}: using model sigmas') + if ddim_num_steps >= self.karras_max: + print(f'>> Ksampler using model noise schedule (steps > {self.karras_max})') self.sigmas = self.model_sigmas else: - print(f'>> number of steps ({ddim_num_steps}) < {STEP_THRESHOLD}: using karras sigmas') + print(f'>> Ksampler using karras noise schedule (steps <= {self.karras_max})') self.sigmas = self.karras_sigmas # ALERT: We are completely overriding the sample() method in the base class, which diff --git a/ldm/models/diffusion/sampler.py b/ldm/models/diffusion/sampler.py index ff705513f8..01193ed5a5 100644 --- a/ldm/models/diffusion/sampler.py +++ b/ldm/models/diffusion/sampler.py @@ -2,8 +2,8 @@ ldm.models.diffusion.sampler Base class for ldm.models.diffusion.ddim, ldm.models.diffusion.ksampler, etc - ''' + import torch import numpy as np from tqdm import tqdm @@ -411,3 +411,15 @@ class Sampler(object): return self.model.inner_model.q_sample(x0,ts) ''' return self.model.q_sample(x0,ts) + + def adjust_settings(self,**kwargs): + ''' + This is a catch-all method for adjusting any instance variables + after the sampler is instantiated. No type-checking performed + here, so use with care! + ''' + for k in kwargs.keys(): + try: + setattr(self,k,kwargs[k]) + except AttributeError: + print(f'** Warning: attempt to set unknown attribute {k} in sampler of type {type(self)}')