diff --git a/.github/workflows/create-caches.yml b/.github/workflows/create-caches.yml index e21286a407..0e0531eaa5 100644 --- a/.github/workflows/create-caches.yml +++ b/.github/workflows/create-caches.yml @@ -1,26 +1,43 @@ name: Create Caches -on: - workflow_dispatch + +on: workflow_dispatch + jobs: - build: + os_matrix: strategy: matrix: - os: [ ubuntu-latest, macos-12 ] - name: Create Caches on ${{ matrix.os }} conda + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + environment-file: environment.yml + default-shell: bash -l {0} + - os: macos-latest + environment-file: environment-mac.yml + default-shell: bash -l {0} + name: Test invoke.py on ${{ matrix.os }} with conda runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.default-shell }} steps: - - name: Set platform variables - id: vars - run: | - if [ "$RUNNER_OS" = "macOS" ]; then - echo "::set-output name=ENV_FILE::environment-mac.yml" - echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" - elif [ "$RUNNER_OS" = "Linux" ]; then - echo "::set-output name=ENV_FILE::environment.yml" - echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" - fi - name: Checkout sources uses: actions/checkout@v3 + + - name: setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-activate-base: false + auto-update-conda: false + miniconda-version: latest + + - name: set environment + run: | + [[ "$GITHUB_REF" == 'refs/heads/main' ]] \ + && echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV \ + || echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV + echo "CONDA_ROOT=$CONDA" >> $GITHUB_ENV + echo "CONDA_ENV_NAME=invokeai" >> $GITHUB_ENV + - name: Use Cached Stable Diffusion v1.4 Model id: cache-sd-v1-4 uses: actions/cache@v3 @@ -29,42 +46,62 @@ jobs: with: path: models/ldm/stable-diffusion-v1/model.ckpt key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} + restore-keys: ${{ env.cache-name }} + + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi - - name: Use Cached Dependencies - id: cache-conda-env-ldm + [[ -d models/ldm/stable-diffusion-v1 ]] \ + || mkdir -p models/ldm/stable-diffusion-v1 + [[ -r models/ldm/stable-diffusion-v1/model.ckpt ]] \ + || curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + + - name: Use cached Conda Environment uses: actions/cache@v3 env: - cache-name: cache-conda-env-ldm + cache-name: cache-conda-env-${{ env.CONDA_ENV_NAME }} + conda-env-file: ${{ matrix.environment-file }} with: - path: ~/.conda/envs/ldm + path: ${{ env.CONDA_ROOT }}/envs/${{ env.CONDA_ENV_NAME }} key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} - - name: Install Dependencies - if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f ${{ steps.vars.outputs.ENV_FILE }} - - name: Use Cached Huggingface and Torch models - id: cache-huggingface-torch + restore-keys: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(env.conda-env-file) }} + + - name: Use cached Conda Packages uses: actions/cache@v3 env: - cache-name: cache-huggingface-torch + cache-name: cache-conda-env-${{ env.CONDA_ENV_NAME }} + conda-env-file: ${{ matrix.environment-file }} + with: + path: ${{ env.CONDA_PKGS_DIR }} + key: ${{ env.cache-name }} + restore-keys: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(env.conda-env-file) }} + + - name: Activate Conda Env + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ${{ env.CONDA_ENV_NAME }} + environment-file: ${{ matrix.environment-file }} + + - name: Use Cached Huggingface and Torch models + id: cache-hugginface-torch + uses: actions/cache@v3 + env: + cache-name: cache-hugginface-torch with: path: ~/.cache key: ${{ env.cache-name }} restore-keys: | ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-huggingface-torch.outputs.cache-hit != 'true' }} - run: | - ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py + + - name: run preload_models.py + run: python scripts/preload_models.py diff --git a/.github/workflows/mkdocs-flow.yml b/.github/workflows/mkdocs-flow.yml deleted file mode 100644 index 318f180c1b..0000000000 --- a/.github/workflows/mkdocs-flow.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Deploy -on: - push: - branches: - - main - # pull_request: - # branches: - # - main -jobs: - build: - name: Deploy docs to GitHub Pages - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Build - uses: Tiryoh/actions-mkdocs@v0 - with: - mkdocs_version: 'latest' # option - requirements: '/requirements-mkdocs.txt' # option - configfile: '/mkdocs.yml' # option - - name: Deploy - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./site diff --git a/.github/workflows/mkdocs-material.yml b/.github/workflows/mkdocs-material.yml new file mode 100644 index 0000000000..2d01dec6df --- /dev/null +++ b/.github/workflows/mkdocs-material.yml @@ -0,0 +1,44 @@ +name: mkdocs-material +on: + push: + branches: + - 'main' + - 'development' + pull_request: + branches: + - 'main' + - 'development' + +jobs: + mkdocs-material: + runs-on: ubuntu-latest + steps: + - name: checkout sources + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: install requirements + run: | + python -m \ + pip install -r requirements-mkdocs.txt + + - name: confirm buildability + run: | + python -m \ + mkdocs build \ + --clean \ + --verbose + + - name: deploy to gh-pages + if: ${{ github.ref == 'refs/heads/main' }} + run: | + python -m \ + mkdocs gh-deploy \ + --clean \ + --force diff --git a/.github/workflows/test-invoke-conda.yml b/.github/workflows/test-invoke-conda.yml index b5314cfd6b..027d1bce47 100644 --- a/.github/workflows/test-invoke-conda.yml +++ b/.github/workflows/test-invoke-conda.yml @@ -4,29 +4,43 @@ on: branches: - 'main' - 'development' + jobs: os_matrix: strategy: matrix: - os: [ ubuntu-latest, macos-12 ] + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + environment-file: environment.yml + default-shell: bash -l {0} + - os: macos-latest + environment-file: environment-mac.yml + default-shell: bash -l {0} name: Test invoke.py on ${{ matrix.os }} with conda runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.default-shell }} steps: - - run: | - echo The PR was merged - - name: Set platform variables - id: vars - run: | - # Note, can't "activate" via github action; specifying the env's python has the same effect - if [ "$RUNNER_OS" = "macOS" ]; then - echo "::set-output name=ENV_FILE::environment-mac.yml" - echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" - elif [ "$RUNNER_OS" = "Linux" ]; then - echo "::set-output name=ENV_FILE::environment.yml" - echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" - fi - name: Checkout sources uses: actions/checkout@v3 + + - name: setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-activate-base: false + auto-update-conda: false + miniconda-version: latest + + - name: set environment + run: | + [[ "$GITHUB_REF" == 'refs/heads/main' ]] \ + && echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV \ + || echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV + echo "CONDA_ROOT=$CONDA" >> $GITHUB_ENV + echo "CONDA_ENV_NAME=invokeai" >> $GITHUB_ENV + - name: Use Cached Stable Diffusion v1.4 Model id: cache-sd-v1-4 uses: actions/cache@v3 @@ -35,31 +49,42 @@ jobs: with: path: models/ldm/stable-diffusion-v1/model.ckpt key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} + restore-keys: ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi - - name: Use Cached Dependencies - id: cache-conda-env-ldm + [[ -d models/ldm/stable-diffusion-v1 ]] \ + || mkdir -p models/ldm/stable-diffusion-v1 + [[ -r models/ldm/stable-diffusion-v1/model.ckpt ]] \ + || curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + + - name: Use cached Conda Environment uses: actions/cache@v3 env: - cache-name: cache-conda-env-ldm + cache-name: cache-conda-env-${{ env.CONDA_ENV_NAME }} + conda-env-file: ${{ matrix.environment-file }} with: - path: ~/.conda/envs/ldm + path: ${{ env.CONDA_ROOT }}/envs/${{ env.CONDA_ENV_NAME }} key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} - - name: Install Dependencies - if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + restore-keys: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(env.conda-env-file) }} + + - name: Use cached Conda Packages + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-${{ env.CONDA_ENV_NAME }} + conda-env-file: ${{ matrix.environment-file }} + with: + path: ${{ env.CONDA_PKGS_DIR }} + key: ${{ env.cache-name }} + restore-keys: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(env.conda-env-file) }} + + - name: Activate Conda Env + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ${{ env.CONDA_ENV_NAME }} + environment-file: ${{ matrix.environment-file }} + - name: Use Cached Huggingface and Torch models id: cache-hugginface-torch uses: actions/cache@v3 @@ -70,28 +95,18 @@ jobs: key: ${{ env.cache-name }} restore-keys: | ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - run: | - ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py -# - name: Run tmate -# uses: mxschmitt/action-tmate@v3 -# timeout-minutes: 30 + + - name: run preload_models.py + run: python scripts/preload_models.py + - name: Run the tests run: | - # Note, can't "activate" via github action; specifying the env's python has the same effect - if [ $(uname) = "Darwin" ]; then - export PYTORCH_ENABLE_MPS_FALLBACK=1 - fi - # Utterly hacky, but I don't know how else to do this - if [[ ${{ github.ref }} == 'refs/heads/master' ]]; then - time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/invoke.py --from_file tests/preflight_prompts.txt - elif [[ ${{ github.ref }} == 'refs/heads/development' ]]; then - time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/invoke.py --from_file tests/dev_prompts.txt - fi + time python scripts/invoke.py \ + --from_file ${{ env.TEST_PROMPTS }} mkdir -p outputs/img-samples + - name: Archive results uses: actions/upload-artifact@v3 with: - name: results + name: results_${{ matrix.os }} path: outputs/img-samples diff --git a/.gitignore b/.gitignore index e556c5ebe6..2663108cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -180,7 +180,7 @@ src **/__pycache__/ outputs -# Logs and associated folders +# Logs and associated folders # created from generated embeddings. logs testtube diff --git a/environment-mac.yml b/environment-mac.yml index c71e8e67e1..735343c508 100644 --- a/environment-mac.yml +++ b/environment-mac.yml @@ -3,56 +3,54 @@ channels: - pytorch - conda-forge dependencies: - - python==3.9.13 - - pip==22.2.2 + - python>=3.10, <3.11 + - pip>=22 # pytorch left unpinned - - pytorch==1.12.1 - - torchvision==0.13.1 - + - pytorch + - torchvision # I suggest to keep the other deps sorted for convenience. # To determine what the latest versions should be, run: # # ```shell - # sed -E 's/ldm/ldm-updated/;20,99s/- ([^=]+)==.+/- \1/' environment-mac.yml > environment-mac-updated.yml - # CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac-updated.yml && conda list -n ldm-updated | awk ' {print " - " $1 "==" $2;} ' + # sed -E 's/invokeai/invokeai-updated/;20,99s/- ([^=]+)==.+/- \1/' environment-mac.yml > environment-mac-updated.yml + # CONDA_SUBDIR=osx-arm64 conda env create -f environment-mac-updated.yml && conda list -n invokeai-updated | awk ' {print " - " $1 "==" $2;} ' # ``` - - albumentations==1.2.1 - - coloredlogs==15.0.1 - - einops==0.4.1 - - grpcio==1.46.4 - - humanfriendly==10.0 - - imageio==2.21.2 - - imageio-ffmpeg==0.4.7 - - imgaug==0.4.0 - - kornia==0.6.7 - - mpmath==1.2.1 - - nomkl=1.0 - - numpy==1.23.2 - - omegaconf==2.1.1 - - openh264==2.3.0 - - onnx==1.12.0 - - onnxruntime==1.12.1 - - pudb==2022.1 - - pytorch-lightning==1.7.5 - - scipy==1.9.1 - - streamlit==1.12.2 - - sympy==1.10.1 - - tensorboard==2.10.0 - - torchmetrics==0.9.3 + - albumentations + - coloredlogs + - einops + - grpcio + - humanfriendly + - imageio + - imageio-ffmpeg + - imgaug + - kornia + - mpmath + - nomkl + - numpy + - omegaconf + - openh264 + - onnx + - onnxruntime + - pudb + - pytorch-lightning + - scipy + - streamlit + - sympy + - tensorboard + - torchmetrics - pip: - - flask==2.1.3 - - flask_socketio==5.3.0 - - flask_cors==3.0.10 - - dependency_injector==4.40.0 - - eventlet==0.33.1 - - opencv-python==4.6.0 - - protobuf==3.19.5 - - realesrgan==0.2.5.0 - - send2trash==1.8.0 - - test-tube==0.7.5 - - transformers==4.21.2 - - torch-fidelity==0.3.0 + - flask + - flask_socketio + - flask_cors + - dependency_injector + - eventlet + - protobuf + - realesrgan + - send2trash + - test-tube + - transformers + - torch-fidelity - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion