From 2433cc344a964352f471fb948a5031c7a8200021 Mon Sep 17 00:00:00 2001
From: Matthias Wild <40327258+mauwii@users.noreply.github.com>
Date: Fri, 25 Nov 2022 01:24:24 +0100
Subject: [PATCH] add test-invoke-pip.yml (#1521)

* add test-invoke-pip.yml

* update requirements-base.txt to fix tests

* install requirements-base.txt separate
since it requires to have torch already installed
also restore origin requirements-base.txt after suc. test in my fork

* restore origin requirements
add `basicsr>=1.4.2` to requirements-base.txt
remove second installation step

* re-add previously overseen req in lin-cuda

* fix typo in setup.py - `scripts/preload_models.py`

* use GFBGAN from branch `basicsr-1.4.2`

* remove `basicsr>=1.4.2` from base reqs

* add INVOKEAI_ROOT to env

* disable upgrade of `pip`, `setuptools` and `wheel`

* try to use a venv which should not contain `wheel`

* add relative path to pip command

* use `configure_invokeai.py --no-interactive --yes`

* set grpcio to `<1.51.0`

* revert changes to use venv

* remove `--prefer-binary`

* disable step to create models.yaml
since this will not be used anymore with new `configure_invokeai.py`

* use `pip install --no-binary=":all:"`

* another try to use venv

* try uninstalling wheel before installing reqs

* dont use requirements.txt as filename

* update cache-dependency-path

* add facexlib to requirements-base.txt

* first install requirements-base.txt

* first install `-e .`, then install requirements
I know that this is obviously the wrong order, but still have a feeling

* add facexlib to requirements.in

* remove `-e .` from reqs and install after reqs

* unpin torch and torchvision in requirements.in

* fix model dl path

* fix curl output path

* create directory before downloading model

* set INVOKEAI_ROOT_PATH
https://docs.github.com/en/actions/learn-github-actions/environment-variables#naming-conventions-for-environment-variables

* INVOKEAI_ROOT ${{ env.GITHUB_WORKSPACE }}/invokeai

* fix matrix stable-diffusion-model-dl-path

* fix INVOKEAI_ROOT

* fix INVOKEAI_ROOT

* add --root and --outdir to run-tests step

* create models.yaml from example

* fix scripts variable in setup.py
by removing unused scripts

* fix archive-results path

* fix workflow to reflect latest code changes

* fix copy paste error

* fix job name

* fix matrix.stable-diffusion-model

* restructure matrix

* fix `activate conda env` step

* update the environment yamls
use same 4 git packages as for pip

* rename job in test-invoke-conda

* add tqdm to environment-lin-amd.yml

* fix python commands in test-invoke-conda.yml

Co-authored-by: Lincoln Stein <lincoln.stein@gmail.com>
---
 .github/workflows/test-invoke-conda.yml       |  69 +++++-----
 .github/workflows/test-invoke-pip.yml         | 125 ++++++++++++++++++
 configs/models.yaml.example                   |   4 +-
 .../environment-lin-amd.yml                   |   5 +-
 .../environment-lin-cuda.yml                  |   4 +-
 .../environment-mac.yml                       |   5 +-
 .../requirements-base.txt                     |   3 +-
 .../requirements-lin-amd.txt                  |   4 +-
 .../requirements-lin-cuda.txt                 |   2 +-
 .../requirements-mac-mps-cpu.txt              |   4 +-
 installer/requirements.in                     |  10 +-
 setup.py                                      |   4 +-
 12 files changed, 186 insertions(+), 53 deletions(-)
 create mode 100644 .github/workflows/test-invoke-pip.yml

diff --git a/.github/workflows/test-invoke-conda.yml b/.github/workflows/test-invoke-conda.yml
index 41838ba565..32126ef7f8 100644
--- a/.github/workflows/test-invoke-conda.yml
+++ b/.github/workflows/test-invoke-conda.yml
@@ -4,6 +4,7 @@ on:
     branches:
       - 'main'
       - 'development'
+      - 'fix-gh-actions-fork'
   pull_request:
     branches:
       - 'main'
@@ -15,28 +16,30 @@ jobs:
       fail-fast: false
       matrix:
         stable-diffusion-model:
-          # - 'https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt'
-          - 'https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt'
-        os:
-          - ubuntu-latest
-          - macOS-12
+          - 'stable-diffusion-1.5'
+        environment-yaml:
+          - environment-lin-amd.yml
+          - environment-lin-cuda.yml
+          - environment-mac.yml
         include:
-          - os: ubuntu-latest
-            environment-file: environment-lin-cuda.yml
+          - environment-yaml: environment-lin-amd.yml
+            os: ubuntu-latest
             default-shell: bash -l {0}
-          - os: macOS-12
-            environment-file: environment-mac.yml
+          - environment-yaml: environment-lin-cuda.yml
+            os: ubuntu-latest
             default-shell: bash -l {0}
-          # - stable-diffusion-model: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt
-          #   stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/sd-v1-4.ckpt
-          #   stable-diffusion-model-switch: stable-diffusion-1.4
-          - stable-diffusion-model: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-switch: stable-diffusion-1.5
-    name: ${{ matrix.os }} with ${{ matrix.stable-diffusion-model-switch }}
+          - environment-yaml: environment-mac.yml
+            os: macos-12
+            default-shell: bash -l {0}
+          - stable-diffusion-model: stable-diffusion-1.5
+            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
+            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
+            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
+    name: ${{ matrix.environment-yaml }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
       CONDA_ENV_NAME: invokeai
+      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
     defaults:
       run:
         shell: ${{ matrix.default-shell }}
@@ -46,17 +49,19 @@ jobs:
         uses: actions/checkout@v3
 
       - name: create models.yaml from example
-        run: cp configs/models.yaml.example configs/models.yaml
+        run: |
+          mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
+          cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
 
       - name: create environment.yml
-        run: cp environments-and-requirements/${{ matrix.environment-file }} environment.yml
+        run: cp "environments-and-requirements/${{ matrix.environment-yaml }}" environment.yml
 
       - name: Use cached conda packages
         id: use-cached-conda-packages
         uses: actions/cache@v3
         with:
           path: ~/conda_pkgs_dir
-          key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-file) }}
+          key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-yaml) }}
 
       - name: Activate Conda Env
         id: activate-conda-env
@@ -82,34 +87,34 @@ jobs:
         id: cache-sd-model
         uses: actions/cache@v3
         env:
-          cache-name: cache-${{ matrix.stable-diffusion-model-switch }}
+          cache-name: cache-${{ matrix.stable-diffusion-model }}
         with:
           path: ${{ matrix.stable-diffusion-model-dl-path }}
           key: ${{ env.cache-name }}
 
-      - name: Download ${{ matrix.stable-diffusion-model-switch }}
+      - name: Download ${{ matrix.stable-diffusion-model }}
         id: download-stable-diffusion-model
         if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
         run: |
-          [[ -d models/ldm/stable-diffusion-v1 ]] \
-            || mkdir -p models/ldm/stable-diffusion-v1
+          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
           curl \
             -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o ${{ matrix.stable-diffusion-model-dl-path }} \
-            -L ${{ matrix.stable-diffusion-model }}
+            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
+            -L ${{ matrix.stable-diffusion-model-url }}
 
-      - name: run preload_models.py
+      - name: run configure_invokeai.py
         id: run-preload-models
         run: |
-          python scripts/preload_models.py \
-            --no-interactive
+          python scripts/configure_invokeai.py --no-interactive --yes
 
       - name: Run the tests
         id: run-tests
         run: |
           time python scripts/invoke.py \
-            --model ${{ matrix.stable-diffusion-model-switch }} \
-            --from_file ${{ env.TEST_PROMPTS }}
+            --model ${{ matrix.stable-diffusion-model }} \
+            --from_file ${{ env.TEST_PROMPTS }} \
+            --root="${{ env.INVOKEAI_ROOT }}" \
+            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
 
       - name: export conda env
         id: export-conda-env
@@ -121,5 +126,5 @@ jobs:
         id: archive-results
         uses: actions/upload-artifact@v3
         with:
-          name: results_${{ matrix.os }}_${{ matrix.stable-diffusion-model-switch }}
-          path: outputs/img-samples
+          name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }}
+          path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples
diff --git a/.github/workflows/test-invoke-pip.yml b/.github/workflows/test-invoke-pip.yml
new file mode 100644
index 0000000000..69fc7c9ce9
--- /dev/null
+++ b/.github/workflows/test-invoke-pip.yml
@@ -0,0 +1,125 @@
+name: Test invoke.py pip
+on:
+  push:
+    branches:
+      - 'main'
+      - 'development'
+  pull_request:
+    branches:
+      - 'main'
+      - 'development'
+
+jobs:
+  matrix:
+    strategy:
+      fail-fast: false
+      matrix:
+        stable-diffusion-model:
+          - stable-diffusion-1.5
+        requirements-file:
+          - requirements-lin-cuda.txt
+          - requirements-lin-amd.txt
+          - requirements-mac-mps-cpu.txt
+        python-version:
+          - '3.9'
+          - '3.10'
+        include:
+          - requirements-file: requirements-lin-cuda.txt
+            os: ubuntu-latest
+            default-shell: bash -l {0}
+          - requirements-file: requirements-lin-amd.txt
+            os: ubuntu-latest
+            default-shell: bash -l {0}
+          - requirements-file: requirements-mac-mps-cpu.txt
+            os: macOS-12
+            default-shell: bash -l {0}
+          - stable-diffusion-model: stable-diffusion-1.5
+            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
+            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
+            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
+    name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: ${{ matrix.default-shell }}
+    env:
+      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+    steps:
+      - name: Checkout sources
+        id: checkout-sources
+        uses: actions/checkout@v3
+
+      - name: create models.yaml from example
+        run: |
+          mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
+          cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
+
+      - name: set test prompt to main branch validation
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV
+
+      - name: set test prompt to development branch validation
+        if: ${{ github.ref == 'refs/heads/development' }}
+        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV
+
+      - name: set test prompt to Pull Request validation
+        if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
+        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV
+
+      - name: create requirements.txt
+        run: cp 'environments-and-requirements/${{ matrix.requirements-file }}' '${{ matrix.requirements-file }}'
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: ${{ matrix.requirements-file }}
+
+      # - name: install dependencies
+      #   run: ${{ env.pythonLocation }}/bin/pip install --upgrade pip setuptools wheel
+
+      - name: install requirements
+        run: |
+          ${{ env.pythonLocation }}/bin/pip install --upgrade -r '${{ matrix.requirements-file }}'
+          ${{ env.pythonLocation }}/bin/pip install -e .
+
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ${{ matrix.stable-diffusion-model-dl-path }}
+          key: ${{ env.cache-name }}
+
+      - name: Download ${{ matrix.stable-diffusion-model }}
+        id: download-stable-diffusion-model
+        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
+          curl \
+            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
+            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
+            -L ${{ matrix.stable-diffusion-model-url }}
+
+      - name: run configure_invokeai.py
+        id: run-preload-models
+        run: |
+          ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py --no-interactive --yes
+
+      - name: Run the tests
+        id: run-tests
+        run: |
+          time ${{ env.pythonLocation }}/bin/python scripts/invoke.py \
+            --model ${{ matrix.stable-diffusion-model }} \
+            --from_file ${{ env.TEST_PROMPTS }} \
+            --root="${{ env.INVOKEAI_ROOT }}" \
+            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
+
+      - name: Archive results
+        id: archive-results
+        uses: actions/upload-artifact@v3
+        with:
+          name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }}
+          path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples
diff --git a/configs/models.yaml.example b/configs/models.yaml.example
index 9c152c25c1..31401cd02d 100644
--- a/configs/models.yaml.example
+++ b/configs/models.yaml.example
@@ -7,8 +7,8 @@
 # was trained on.
 stable-diffusion-1.5:
   description: The newest Stable Diffusion version 1.5 weight file (4.27 GB)
-  weights: ./models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
-  config: ./configs/stable-diffusion/v1-inference.yaml
+  weights: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
+  config: configs/stable-diffusion/v1-inference.yaml
   width: 512
   height: 512
   vae: ./models/ldm/stable-diffusion-v1/vae-ft-mse-840000-ema-pruned.ckpt
diff --git a/environments-and-requirements/environment-lin-amd.yml b/environments-and-requirements/environment-lin-amd.yml
index 15a8b9b0db..8388a84206 100644
--- a/environments-and-requirements/environment-lin-amd.yml
+++ b/environments-and-requirements/environment-lin-amd.yml
@@ -33,6 +33,7 @@ dependencies:
     - streamlit==1.12.0
     - taming-transformers-rom1504
     - test-tube>=0.7.5
+    - tqdm
     - torch
     - torch-fidelity==0.3.0
     - torchaudio
@@ -40,7 +41,7 @@ dependencies:
     - torchvision
     - transformers==4.21.3
     - git+https://github.com/openai/CLIP.git@main#egg=clip
-    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion
+    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
     - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-    - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+    - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
     - -e .
diff --git a/environments-and-requirements/environment-lin-cuda.yml b/environments-and-requirements/environment-lin-cuda.yml
index b1926831f1..895515d43a 100644
--- a/environments-and-requirements/environment-lin-cuda.yml
+++ b/environments-and-requirements/environment-lin-cuda.yml
@@ -40,7 +40,7 @@ dependencies:
     - torchmetrics==0.7.0
     - transformers==4.21.3
     - git+https://github.com/openai/CLIP.git@main#egg=clip
-    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion
+    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
     - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-    - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+    - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
     - -e .
diff --git a/environments-and-requirements/environment-mac.yml b/environments-and-requirements/environment-mac.yml
index 4fe79d453a..9bf8e27952 100644
--- a/environments-and-requirements/environment-mac.yml
+++ b/environments-and-requirements/environment-mac.yml
@@ -56,10 +56,9 @@ dependencies:
       - taming-transformers-rom1504
       - test-tube==0.7.5
       - git+https://github.com/openai/CLIP.git@main#egg=clip
-      - git+https://github.com/invoke-ai/k-diffusion.git@mps#egg=k_diffusion
-      - git+https://github.com/invoke-ai/Real-ESRGAN.git#egg=realesrgan
-      - git+https://github.com/invoke-ai/GFPGAN.git#egg=gfpgan
+      - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
       - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
+      - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
       - -e .
 variables:
   PYTORCH_ENABLE_MPS_FALLBACK: 1
diff --git a/environments-and-requirements/requirements-base.txt b/environments-and-requirements/requirements-base.txt
index d76531a133..a59cdee138 100644
--- a/environments-and-requirements/requirements-base.txt
+++ b/environments-and-requirements/requirements-base.txt
@@ -4,6 +4,7 @@ dependency_injector==4.40.0
 diffusers
 einops
 eventlet
+facexlib
 flask==2.1.3
 flask_cors==3.0.10
 flask_socketio==5.3.0
@@ -34,4 +35,4 @@ picklescan
 git+https://github.com/openai/CLIP.git@main#egg=clip
 git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
 git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
diff --git a/environments-and-requirements/requirements-lin-amd.txt b/environments-and-requirements/requirements-lin-amd.txt
index 8ca0ead88c..1687ef0996 100644
--- a/environments-and-requirements/requirements-lin-amd.txt
+++ b/environments-and-requirements/requirements-lin-amd.txt
@@ -1,7 +1,7 @@
 -r environments-and-requirements/requirements-base.txt
 
-# Get hardware-appropriate torch/torchvision 
+# Get hardware-appropriate torch/torchvision
 --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org
 torch
 torchvision
--e .
+# -e .
diff --git a/environments-and-requirements/requirements-lin-cuda.txt b/environments-and-requirements/requirements-lin-cuda.txt
index 1578c60334..06e6e73366 100644
--- a/environments-and-requirements/requirements-lin-cuda.txt
+++ b/environments-and-requirements/requirements-lin-cuda.txt
@@ -1,2 +1,2 @@
 -r environments-and-requirements/requirements-base.txt
--e .
+# -e .
diff --git a/environments-and-requirements/requirements-mac-mps-cpu.txt b/environments-and-requirements/requirements-mac-mps-cpu.txt
index 531d57a307..54071e73c7 100644
--- a/environments-and-requirements/requirements-mac-mps-cpu.txt
+++ b/environments-and-requirements/requirements-mac-mps-cpu.txt
@@ -1,7 +1,7 @@
 -r environments-and-requirements/requirements-base.txt
 
-grpcio==1.50.0
+grpcio<1.51.0
 protobuf==3.19.6
 torch<1.13.0
 torchvision<0.14.0
--e .
+# -e .
diff --git a/installer/requirements.in b/installer/requirements.in
index 3eb796e0f0..dab06aa72d 100644
--- a/installer/requirements.in
+++ b/installer/requirements.in
@@ -4,12 +4,16 @@
 accelerate~=0.14
 albumentations
 diffusers
+einops
 eventlet
+facexlib
 flask_cors
 flask_socketio
 flaskwebgui
 getpass_asterisk
 imageio-ffmpeg
+kornia
+numpy
 pyreadline3
 realesrgan
 send2trash
@@ -17,10 +21,8 @@ streamlit
 taming-transformers-rom1504
 test-tube
 torch-fidelity
-torch==1.12.1 ; platform_system == 'Darwin'
-torch==1.12.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows'
-torchvision==0.13.0 ; platform_system == 'Darwin'
-torchvision==0.13.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows'
+torch
+torchvision
 transformers
 picklescan
 https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip
diff --git a/setup.py b/setup.py
index d40100e318..cef2e6cc3e 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@ def _get_requirements(path):
             packages = f.read().splitlines()
     except (IOError, OSError) as ex:
         raise RuntimeError("Can't open file with requirements: %s", repr(ex))
- 
+
     # Drop option lines
     packages = [package for package in packages if not re.match(r"^--", package)]
     packages = [package for package in packages if not re.match(r"^http", package)]
@@ -70,7 +70,7 @@ setup(
         'Topic :: Scientific/Engineering :: Artificial Intelligence',
         'Topic :: Scientific/Engineering :: Image Processing',
     ],
-    scripts = ['scripts/invoke.py','scripts/configure_invokeai.py','scripts/sd-metadata.py'],
+    scripts = ['scripts/invoke.py','scripts/configure_invokeai.py', 'scripts/sd-metadata.py'],
     data_files=[('frontend',frontend_files)],
 )