Merge branch 'development' into feat-install-unify-setup-requirements-pip

2024-08-30 20:32:17 +00:00 · 2022-11-24 21:55:39 -05:00 · 2022-11-24 21:55:39 -05:00 · 461358bdde
commit 461358bdde
parent 5a1a36ec29 2433cc344a
13 changed files with 191 additions and 56 deletions
--- a/.github/workflows/test-invoke-conda.yml
+++ b/.github/workflows/test-invoke-conda.yml
@ -4,6 +4,7 @@ on:
    branches:
      - 'main'
      - 'development'
+      - 'fix-gh-actions-fork'
  pull_request:
    branches:
      - 'main'
@ -15,28 +16,30 @@ jobs:
      fail-fast: false
      matrix:
        stable-diffusion-model:
-          # - 'https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt'
-          - 'https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt'
-        os:
-          - ubuntu-latest
-          - macOS-12
+          - 'stable-diffusion-1.5'
+        environment-yaml:
+          - environment-lin-amd.yml
+          - environment-lin-cuda.yml
+          - environment-mac.yml
        include:
-          - os: ubuntu-latest
-            environment-file: environment-lin-cuda.yml
+          - environment-yaml: environment-lin-amd.yml
+            os: ubuntu-latest
            default-shell: bash -l {0}
-          - os: macOS-12
-            environment-file: environment-mac.yml
+          - environment-yaml: environment-lin-cuda.yml
+            os: ubuntu-latest
            default-shell: bash -l {0}
-          # - stable-diffusion-model: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt
-          #   stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/sd-v1-4.ckpt
-          #   stable-diffusion-model-switch: stable-diffusion-1.4
-          - stable-diffusion-model: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-switch: stable-diffusion-1.5
-    name: ${{ matrix.os }} with ${{ matrix.stable-diffusion-model-switch }}
+          - environment-yaml: environment-mac.yml
+            os: macos-12
+            default-shell: bash -l {0}
+          - stable-diffusion-model: stable-diffusion-1.5
+            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
+            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
+            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
+    name: ${{ matrix.environment-yaml }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    env:
      CONDA_ENV_NAME: invokeai
+      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
    defaults:
      run:
        shell: ${{ matrix.default-shell }}
@ -46,17 +49,19 @@ jobs:
        uses: actions/checkout@v3

      - name: create models.yaml from example
-        run: cp configs/models.yaml.example configs/models.yaml
+        run: |
+          mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
+          cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml

      - name: create environment.yml
-        run: cp environments-and-requirements/${{ matrix.environment-file }} environment.yml
+        run: cp "environments-and-requirements/${{ matrix.environment-yaml }}" environment.yml

      - name: Use cached conda packages
        id: use-cached-conda-packages
        uses: actions/cache@v3
        with:
          path: ~/conda_pkgs_dir
-          key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-file) }}
+          key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-yaml) }}

      - name: Activate Conda Env
        id: activate-conda-env
@ -82,34 +87,34 @@ jobs:
        id: cache-sd-model
        uses: actions/cache@v3
        env:
-          cache-name: cache-${{ matrix.stable-diffusion-model-switch }}
+          cache-name: cache-${{ matrix.stable-diffusion-model }}
        with:
          path: ${{ matrix.stable-diffusion-model-dl-path }}
          key: ${{ env.cache-name }}

-      - name: Download ${{ matrix.stable-diffusion-model-switch }}
+      - name: Download ${{ matrix.stable-diffusion-model }}
        id: download-stable-diffusion-model
        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
        run: |
-          [[ -d models/ldm/stable-diffusion-v1 ]] \
-            || mkdir -p models/ldm/stable-diffusion-v1
+          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
          curl \
            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o ${{ matrix.stable-diffusion-model-dl-path }} \
-            -L ${{ matrix.stable-diffusion-model }}
+            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
+            -L ${{ matrix.stable-diffusion-model-url }}

-      - name: run preload_models.py
+      - name: run configure_invokeai.py
        id: run-preload-models
        run: |
-          python scripts/preload_models.py \
-            --no-interactive
+          python scripts/configure_invokeai.py --no-interactive --yes

      - name: Run the tests
        id: run-tests
        run: |
          time python scripts/invoke.py \
-            --model ${{ matrix.stable-diffusion-model-switch }} \
-            --from_file ${{ env.TEST_PROMPTS }}
+            --model ${{ matrix.stable-diffusion-model }} \
+            --from_file ${{ env.TEST_PROMPTS }} \
+            --root="${{ env.INVOKEAI_ROOT }}" \
+            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"

      - name: export conda env
        id: export-conda-env
@ -121,5 +126,5 @@ jobs:
        id: archive-results
        uses: actions/upload-artifact@v3
        with:
-          name: results_${{ matrix.os }}_${{ matrix.stable-diffusion-model-switch }}
-          path: outputs/img-samples
+          name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }}
+          path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples
--- a/.github/workflows/test-invoke-pip.yml
+++ b/.github/workflows/test-invoke-pip.yml
@ -0,0 +1,125 @@
+name: Test invoke.py pip
+on:
+  push:
+    branches:
+      - 'main'
+      - 'development'
+  pull_request:
+    branches:
+      - 'main'
+      - 'development'
+
+jobs:
+  matrix:
+    strategy:
+      fail-fast: false
+      matrix:
+        stable-diffusion-model:
+          - stable-diffusion-1.5
+        requirements-file:
+          - requirements-lin-cuda.txt
+          - requirements-lin-amd.txt
+          - requirements-mac-mps-cpu.txt
+        python-version:
+          - '3.9'
+          - '3.10'
+        include:
+          - requirements-file: requirements-lin-cuda.txt
+            os: ubuntu-latest
+            default-shell: bash -l {0}
+          - requirements-file: requirements-lin-amd.txt
+            os: ubuntu-latest
+            default-shell: bash -l {0}
+          - requirements-file: requirements-mac-mps-cpu.txt
+            os: macOS-12
+            default-shell: bash -l {0}
+          - stable-diffusion-model: stable-diffusion-1.5
+            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
+            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
+            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
+    name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: ${{ matrix.default-shell }}
+    env:
+      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+    steps:
+      - name: Checkout sources
+        id: checkout-sources
+        uses: actions/checkout@v3
+
+      - name: create models.yaml from example
+        run: |
+          mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
+          cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
+
+      - name: set test prompt to main branch validation
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV
+
+      - name: set test prompt to development branch validation
+        if: ${{ github.ref == 'refs/heads/development' }}
+        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV
+
+      - name: set test prompt to Pull Request validation
+        if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
+        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV
+
+      - name: create requirements.txt
+        run: cp 'environments-and-requirements/${{ matrix.requirements-file }}' '${{ matrix.requirements-file }}'
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: ${{ matrix.requirements-file }}
+
+      # - name: install dependencies
+      #   run: ${{ env.pythonLocation }}/bin/pip install --upgrade pip setuptools wheel
+
+      - name: install requirements
+        run: |
+          ${{ env.pythonLocation }}/bin/pip install --upgrade -r '${{ matrix.requirements-file }}'
+          ${{ env.pythonLocation }}/bin/pip install -e .
+
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ${{ matrix.stable-diffusion-model-dl-path }}
+          key: ${{ env.cache-name }}
+
+      - name: Download ${{ matrix.stable-diffusion-model }}
+        id: download-stable-diffusion-model
+        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
+          curl \
+            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
+            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
+            -L ${{ matrix.stable-diffusion-model-url }}
+
+      - name: run configure_invokeai.py
+        id: run-preload-models
+        run: |
+          ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py --no-interactive --yes
+
+      - name: Run the tests
+        id: run-tests
+        run: |
+          time ${{ env.pythonLocation }}/bin/python scripts/invoke.py \
+            --model ${{ matrix.stable-diffusion-model }} \
+            --from_file ${{ env.TEST_PROMPTS }} \
+            --root="${{ env.INVOKEAI_ROOT }}" \
+            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
+
+      - name: Archive results
+        id: archive-results
+        uses: actions/upload-artifact@v3
+        with:
+          name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }}
+          path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples
--- a/configs/models.yaml.example
+++ b/configs/models.yaml.example
@ -7,8 +7,8 @@
 # was trained on.
 stable-diffusion-1.5:
  description: The newest Stable Diffusion version 1.5 weight file (4.27 GB)
-  weights: ./models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
-  config: ./configs/stable-diffusion/v1-inference.yaml
+  weights: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt
+  config: configs/stable-diffusion/v1-inference.yaml
  width: 512
  height: 512
  vae: ./models/ldm/stable-diffusion-v1/vae-ft-mse-840000-ema-pruned.ckpt
--- a/environments-and-requirements/environment-lin-amd.yml
+++ b/environments-and-requirements/environment-lin-amd.yml
@ -33,6 +33,7 @@ dependencies:
    - streamlit==1.12.0
    - taming-transformers-rom1504
    - test-tube>=0.7.5
+    - tqdm
    - torch
    - torch-fidelity==0.3.0
    - torchaudio
@ -40,7 +41,7 @@ dependencies:
    - torchvision
    - transformers==4.21.3
    - git+https://github.com/openai/CLIP.git@main#egg=clip
-    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion
+    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
    - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-    - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+    - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
    - -e .
--- a/environments-and-requirements/environment-lin-cuda.yml
+++ b/environments-and-requirements/environment-lin-cuda.yml
@ -40,7 +40,7 @@ dependencies:
    - torchmetrics==0.7.0
    - transformers==4.21.3
    - git+https://github.com/openai/CLIP.git@main#egg=clip
-    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion
+    - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
    - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-    - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+    - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
    - -e .
--- a/environments-and-requirements/environment-mac.yml
+++ b/environments-and-requirements/environment-mac.yml
@ -56,10 +56,9 @@ dependencies:
      - taming-transformers-rom1504
      - test-tube==0.7.5
      - git+https://github.com/openai/CLIP.git@main#egg=clip
-      - git+https://github.com/invoke-ai/k-diffusion.git@mps#egg=k_diffusion
-      - git+https://github.com/invoke-ai/Real-ESRGAN.git#egg=realesrgan
-      - git+https://github.com/invoke-ai/GFPGAN.git#egg=gfpgan
+      - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
      - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
+      - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
      - -e .
 variables:
  PYTORCH_ENABLE_MPS_FALLBACK: 1
--- a/environments-and-requirements/requirements-base.txt
+++ b/environments-and-requirements/requirements-base.txt
@ -4,6 +4,7 @@ dependency_injector==4.40.0
 diffusers
 einops
 eventlet
+facexlib
 flask==2.1.3
 flask_cors==3.0.10
 flask_socketio==5.3.0
@ -34,4 +35,4 @@ picklescan
 git+https://github.com/openai/CLIP.git@main#egg=clip
 git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
 git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
-git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan
+git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan
--- a/environments-and-requirements/requirements-lin-amd.txt
+++ b/environments-and-requirements/requirements-lin-amd.txt
@ -1,7 +1,7 @@
 -r environments-and-requirements/requirements-base.txt

-# Get hardware-appropriate torch/torchvision 
+# Get hardware-appropriate torch/torchvision
 --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org
 torch
 torchvision
-e .
+# -e .
--- a/environments-and-requirements/requirements-lin-cuda.txt
+++ b/environments-and-requirements/requirements-lin-cuda.txt
@ -1,2 +1,2 @@
 -r environments-and-requirements/requirements-base.txt
-e .
+# -e .
--- a/environments-and-requirements/requirements-mac-mps-cpu.txt
+++ b/environments-and-requirements/requirements-mac-mps-cpu.txt
@ -1,7 +1,7 @@
 -r environments-and-requirements/requirements-base.txt

-grpcio==1.50.0
+grpcio<1.51.0
 protobuf==3.19.6
 torch<1.13.0
 torchvision<0.14.0
-e .
+# -e .
--- a/installer/requirements.in
+++ b/installer/requirements.in
@ -4,12 +4,16 @@
 accelerate~=0.14
 albumentations
 diffusers
+einops
 eventlet
+facexlib
 flask_cors
 flask_socketio
 flaskwebgui
 getpass_asterisk
 imageio-ffmpeg
+kornia
+numpy
 pyreadline3
 realesrgan
 send2trash
@ -17,10 +21,8 @@ streamlit
 taming-transformers-rom1504
 test-tube
 torch-fidelity
-torch==1.12.1 ; platform_system == 'Darwin'
-torch==1.12.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows'
-torchvision==0.13.0 ; platform_system == 'Darwin'
-torchvision==0.13.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows'
+torch
+torchvision
 transformers
 picklescan
 clip @ https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip
--- a/scripts/configure_invokeai.py
+++ b/scripts/configure_invokeai.py
@ -594,11 +594,11 @@ def initialize_rootdir(root:str,yes_to_all:bool=False):
    print(f'You may also change the runtime directory by setting the environment variable INVOKEAI_ROOT.\n')
    for name in ('models','configs','scripts','frontend/dist'):
        os.makedirs(os.path.join(root,name), exist_ok=True)
-    for src in ('configs','scripts','frontend/dist'):
+    for src in ['configs']:
        dest = os.path.join(root,src)
        if not os.path.samefile(src,dest):
            shutil.copytree(src,dest,dirs_exist_ok=True)
-    os.makedirs(outputs)
+    os.makedirs(outputs, exist_ok=True)

    init_file = os.path.expanduser(Globals.initfile)
    if not os.path.exists(init_file):
@ -679,7 +679,7 @@ def main():
        if Globals.root == '' \
           or not os.path.exists(os.path.join(Globals.root,'configs/stable-diffusion/v1-inference.yaml')) \
           or not os.path.exists(os.path.join(Globals.root,'frontend/dist')):
-            initialize_rootdir(Globals.root,opt.yes_to_all)
+            initialize_rootdir(Globals.root,(not opt.interactive) or opt.yes_to_all)

        print(f'(Initializing with runtime root {Globals.root})\n')

--- a/setup.py
+++ b/setup.py
@ -16,8 +16,8 @@ def _get_requirements(path):
    try:
        with open(path) as f:
            packages = f.read().splitlines()
-    except OSError as ex:
-        raise RuntimeError('Cannot open file with requirements: %s', repr(ex))
+    except (IOError, OSError) as ex:
+        raise RuntimeError("Can't open file with requirements: %s", repr(ex))

    # Drop option lines
    packages = [package for package in packages if not re.match(r'^--', package)]
@ -74,4 +74,6 @@ setup(
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Scientific/Engineering :: Image Processing',
    ],
+    scripts = ['scripts/invoke.py','scripts/configure_invokeai.py', 'scripts/preload_models.py', 'scripts/sd-metadata.py'],
+    data_files=[('frontend',frontend_files)],
 )