diff --git a/.github/workflows/test-invoke-conda.yml b/.github/workflows/test-invoke-conda.yml index 41838ba565..32126ef7f8 100644 --- a/.github/workflows/test-invoke-conda.yml +++ b/.github/workflows/test-invoke-conda.yml @@ -4,6 +4,7 @@ on: branches: - 'main' - 'development' + - 'fix-gh-actions-fork' pull_request: branches: - 'main' @@ -15,28 +16,30 @@ jobs: fail-fast: false matrix: stable-diffusion-model: - # - 'https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt' - - 'https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt' - os: - - ubuntu-latest - - macOS-12 + - 'stable-diffusion-1.5' + environment-yaml: + - environment-lin-amd.yml + - environment-lin-cuda.yml + - environment-mac.yml include: - - os: ubuntu-latest - environment-file: environment-lin-cuda.yml + - environment-yaml: environment-lin-amd.yml + os: ubuntu-latest default-shell: bash -l {0} - - os: macOS-12 - environment-file: environment-mac.yml + - environment-yaml: environment-lin-cuda.yml + os: ubuntu-latest default-shell: bash -l {0} - # - stable-diffusion-model: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt - # stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/sd-v1-4.ckpt - # stable-diffusion-model-switch: stable-diffusion-1.4 - - stable-diffusion-model: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt - stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt - stable-diffusion-model-switch: stable-diffusion-1.5 - name: ${{ matrix.os }} with ${{ matrix.stable-diffusion-model-switch }} + - environment-yaml: environment-mac.yml + os: macos-12 + default-shell: bash -l {0} + - stable-diffusion-model: stable-diffusion-1.5 + stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt + stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1 + stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt + name: ${{ matrix.environment-yaml }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} env: CONDA_ENV_NAME: invokeai + INVOKEAI_ROOT: '${{ github.workspace }}/invokeai' defaults: run: shell: ${{ matrix.default-shell }} @@ -46,17 +49,19 @@ jobs: uses: actions/checkout@v3 - name: create models.yaml from example - run: cp configs/models.yaml.example configs/models.yaml + run: | + mkdir -p ${{ env.INVOKEAI_ROOT }}/configs + cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml - name: create environment.yml - run: cp environments-and-requirements/${{ matrix.environment-file }} environment.yml + run: cp "environments-and-requirements/${{ matrix.environment-yaml }}" environment.yml - name: Use cached conda packages id: use-cached-conda-packages uses: actions/cache@v3 with: path: ~/conda_pkgs_dir - key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-file) }} + key: conda-pkgs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(matrix.environment-yaml) }} - name: Activate Conda Env id: activate-conda-env @@ -82,34 +87,34 @@ jobs: id: cache-sd-model uses: actions/cache@v3 env: - cache-name: cache-${{ matrix.stable-diffusion-model-switch }} + cache-name: cache-${{ matrix.stable-diffusion-model }} with: path: ${{ matrix.stable-diffusion-model-dl-path }} key: ${{ env.cache-name }} - - name: Download ${{ matrix.stable-diffusion-model-switch }} + - name: Download ${{ matrix.stable-diffusion-model }} id: download-stable-diffusion-model if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }} run: | - [[ -d models/ldm/stable-diffusion-v1 ]] \ - || mkdir -p models/ldm/stable-diffusion-v1 + mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}" curl \ -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \ - -o ${{ matrix.stable-diffusion-model-dl-path }} \ - -L ${{ matrix.stable-diffusion-model }} + -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \ + -L ${{ matrix.stable-diffusion-model-url }} - - name: run preload_models.py + - name: run configure_invokeai.py id: run-preload-models run: | - python scripts/preload_models.py \ - --no-interactive + python scripts/configure_invokeai.py --no-interactive --yes - name: Run the tests id: run-tests run: | time python scripts/invoke.py \ - --model ${{ matrix.stable-diffusion-model-switch }} \ - --from_file ${{ env.TEST_PROMPTS }} + --model ${{ matrix.stable-diffusion-model }} \ + --from_file ${{ env.TEST_PROMPTS }} \ + --root="${{ env.INVOKEAI_ROOT }}" \ + --outdir="${{ env.INVOKEAI_ROOT }}/outputs" - name: export conda env id: export-conda-env @@ -121,5 +126,5 @@ jobs: id: archive-results uses: actions/upload-artifact@v3 with: - name: results_${{ matrix.os }}_${{ matrix.stable-diffusion-model-switch }} - path: outputs/img-samples + name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }} + path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples diff --git a/.github/workflows/test-invoke-pip.yml b/.github/workflows/test-invoke-pip.yml new file mode 100644 index 0000000000..69fc7c9ce9 --- /dev/null +++ b/.github/workflows/test-invoke-pip.yml @@ -0,0 +1,125 @@ +name: Test invoke.py pip +on: + push: + branches: + - 'main' + - 'development' + pull_request: + branches: + - 'main' + - 'development' + +jobs: + matrix: + strategy: + fail-fast: false + matrix: + stable-diffusion-model: + - stable-diffusion-1.5 + requirements-file: + - requirements-lin-cuda.txt + - requirements-lin-amd.txt + - requirements-mac-mps-cpu.txt + python-version: + - '3.9' + - '3.10' + include: + - requirements-file: requirements-lin-cuda.txt + os: ubuntu-latest + default-shell: bash -l {0} + - requirements-file: requirements-lin-amd.txt + os: ubuntu-latest + default-shell: bash -l {0} + - requirements-file: requirements-mac-mps-cpu.txt + os: macOS-12 + default-shell: bash -l {0} + - stable-diffusion-model: stable-diffusion-1.5 + stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt + stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1 + stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt + name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.default-shell }} + env: + INVOKEAI_ROOT: '${{ github.workspace }}/invokeai' + steps: + - name: Checkout sources + id: checkout-sources + uses: actions/checkout@v3 + + - name: create models.yaml from example + run: | + mkdir -p ${{ env.INVOKEAI_ROOT }}/configs + cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml + + - name: set test prompt to main branch validation + if: ${{ github.ref == 'refs/heads/main' }} + run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV + + - name: set test prompt to development branch validation + if: ${{ github.ref == 'refs/heads/development' }} + run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV + + - name: set test prompt to Pull Request validation + if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }} + run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV + + - name: create requirements.txt + run: cp 'environments-and-requirements/${{ matrix.requirements-file }}' '${{ matrix.requirements-file }}' + + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: ${{ matrix.requirements-file }} + + # - name: install dependencies + # run: ${{ env.pythonLocation }}/bin/pip install --upgrade pip setuptools wheel + + - name: install requirements + run: | + ${{ env.pythonLocation }}/bin/pip install --upgrade -r '${{ matrix.requirements-file }}' + ${{ env.pythonLocation }}/bin/pip install -e . + + - name: Use Cached Stable Diffusion Model + id: cache-sd-model + uses: actions/cache@v3 + env: + cache-name: cache-${{ matrix.stable-diffusion-model }} + with: + path: ${{ matrix.stable-diffusion-model-dl-path }} + key: ${{ env.cache-name }} + + - name: Download ${{ matrix.stable-diffusion-model }} + id: download-stable-diffusion-model + if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }} + run: | + mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}" + curl \ + -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \ + -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \ + -L ${{ matrix.stable-diffusion-model-url }} + + - name: run configure_invokeai.py + id: run-preload-models + run: | + ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py --no-interactive --yes + + - name: Run the tests + id: run-tests + run: | + time ${{ env.pythonLocation }}/bin/python scripts/invoke.py \ + --model ${{ matrix.stable-diffusion-model }} \ + --from_file ${{ env.TEST_PROMPTS }} \ + --root="${{ env.INVOKEAI_ROOT }}" \ + --outdir="${{ env.INVOKEAI_ROOT }}/outputs" + + - name: Archive results + id: archive-results + uses: actions/upload-artifact@v3 + with: + name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }} + path: ${{ env.INVOKEAI_ROOT }}/outputs/img-samples diff --git a/configs/models.yaml.example b/configs/models.yaml.example index 9c152c25c1..31401cd02d 100644 --- a/configs/models.yaml.example +++ b/configs/models.yaml.example @@ -7,8 +7,8 @@ # was trained on. stable-diffusion-1.5: description: The newest Stable Diffusion version 1.5 weight file (4.27 GB) - weights: ./models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt - config: ./configs/stable-diffusion/v1-inference.yaml + weights: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt + config: configs/stable-diffusion/v1-inference.yaml width: 512 height: 512 vae: ./models/ldm/stable-diffusion-v1/vae-ft-mse-840000-ema-pruned.ckpt diff --git a/environments-and-requirements/environment-lin-amd.yml b/environments-and-requirements/environment-lin-amd.yml index 15a8b9b0db..8388a84206 100644 --- a/environments-and-requirements/environment-lin-amd.yml +++ b/environments-and-requirements/environment-lin-amd.yml @@ -33,6 +33,7 @@ dependencies: - streamlit==1.12.0 - taming-transformers-rom1504 - test-tube>=0.7.5 + - tqdm - torch - torch-fidelity==0.3.0 - torchaudio @@ -40,7 +41,7 @@ dependencies: - torchvision - transformers==4.21.3 - git+https://github.com/openai/CLIP.git@main#egg=clip - - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion + - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg - - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan + - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan - -e . diff --git a/environments-and-requirements/environment-lin-cuda.yml b/environments-and-requirements/environment-lin-cuda.yml index b1926831f1..895515d43a 100644 --- a/environments-and-requirements/environment-lin-cuda.yml +++ b/environments-and-requirements/environment-lin-cuda.yml @@ -40,7 +40,7 @@ dependencies: - torchmetrics==0.7.0 - transformers==4.21.3 - git+https://github.com/openai/CLIP.git@main#egg=clip - - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion + - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg - - git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan + - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan - -e . diff --git a/environments-and-requirements/environment-mac.yml b/environments-and-requirements/environment-mac.yml index 4fe79d453a..9bf8e27952 100644 --- a/environments-and-requirements/environment-mac.yml +++ b/environments-and-requirements/environment-mac.yml @@ -56,10 +56,9 @@ dependencies: - taming-transformers-rom1504 - test-tube==0.7.5 - git+https://github.com/openai/CLIP.git@main#egg=clip - - git+https://github.com/invoke-ai/k-diffusion.git@mps#egg=k_diffusion - - git+https://github.com/invoke-ai/Real-ESRGAN.git#egg=realesrgan - - git+https://github.com/invoke-ai/GFPGAN.git#egg=gfpgan + - git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion - git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg + - git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan - -e . variables: PYTORCH_ENABLE_MPS_FALLBACK: 1 diff --git a/environments-and-requirements/requirements-base.txt b/environments-and-requirements/requirements-base.txt index d76531a133..a59cdee138 100644 --- a/environments-and-requirements/requirements-base.txt +++ b/environments-and-requirements/requirements-base.txt @@ -4,6 +4,7 @@ dependency_injector==4.40.0 diffusers einops eventlet +facexlib flask==2.1.3 flask_cors==3.0.10 flask_socketio==5.3.0 @@ -34,4 +35,4 @@ picklescan git+https://github.com/openai/CLIP.git@main#egg=clip git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg -git+https://github.com/invoke-ai/GFPGAN#egg=gfpgan +git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan diff --git a/environments-and-requirements/requirements-lin-amd.txt b/environments-and-requirements/requirements-lin-amd.txt index 8ca0ead88c..1687ef0996 100644 --- a/environments-and-requirements/requirements-lin-amd.txt +++ b/environments-and-requirements/requirements-lin-amd.txt @@ -1,7 +1,7 @@ -r environments-and-requirements/requirements-base.txt -# Get hardware-appropriate torch/torchvision +# Get hardware-appropriate torch/torchvision --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org torch torchvision --e . +# -e . diff --git a/environments-and-requirements/requirements-lin-cuda.txt b/environments-and-requirements/requirements-lin-cuda.txt index 1578c60334..06e6e73366 100644 --- a/environments-and-requirements/requirements-lin-cuda.txt +++ b/environments-and-requirements/requirements-lin-cuda.txt @@ -1,2 +1,2 @@ -r environments-and-requirements/requirements-base.txt --e . +# -e . diff --git a/environments-and-requirements/requirements-mac-mps-cpu.txt b/environments-and-requirements/requirements-mac-mps-cpu.txt index 531d57a307..54071e73c7 100644 --- a/environments-and-requirements/requirements-mac-mps-cpu.txt +++ b/environments-and-requirements/requirements-mac-mps-cpu.txt @@ -1,7 +1,7 @@ -r environments-and-requirements/requirements-base.txt -grpcio==1.50.0 +grpcio<1.51.0 protobuf==3.19.6 torch<1.13.0 torchvision<0.14.0 --e . +# -e . diff --git a/installer/requirements.in b/installer/requirements.in index 894ce85b6c..d521b1c677 100644 --- a/installer/requirements.in +++ b/installer/requirements.in @@ -4,12 +4,16 @@ accelerate~=0.14 albumentations diffusers +einops eventlet +facexlib flask_cors flask_socketio flaskwebgui getpass_asterisk imageio-ffmpeg +kornia +numpy pyreadline3 realesrgan send2trash @@ -17,10 +21,8 @@ streamlit taming-transformers-rom1504 test-tube torch-fidelity -torch==1.12.1 ; platform_system == 'Darwin' -torch==1.12.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows' -torchvision==0.13.0 ; platform_system == 'Darwin' -torchvision==0.13.0+cu116 ; platform_system == 'Linux' or platform_system == 'Windows' +torch +torchvision transformers picklescan clip @ https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip diff --git a/scripts/configure_invokeai.py b/scripts/configure_invokeai.py index 7694b1c710..f811df6e97 100755 --- a/scripts/configure_invokeai.py +++ b/scripts/configure_invokeai.py @@ -594,11 +594,11 @@ def initialize_rootdir(root:str,yes_to_all:bool=False): print(f'You may also change the runtime directory by setting the environment variable INVOKEAI_ROOT.\n') for name in ('models','configs','scripts','frontend/dist'): os.makedirs(os.path.join(root,name), exist_ok=True) - for src in ('configs','scripts','frontend/dist'): + for src in ['configs']: dest = os.path.join(root,src) if not os.path.samefile(src,dest): shutil.copytree(src,dest,dirs_exist_ok=True) - os.makedirs(outputs) + os.makedirs(outputs, exist_ok=True) init_file = os.path.expanduser(Globals.initfile) if not os.path.exists(init_file): @@ -679,7 +679,7 @@ def main(): if Globals.root == '' \ or not os.path.exists(os.path.join(Globals.root,'configs/stable-diffusion/v1-inference.yaml')) \ or not os.path.exists(os.path.join(Globals.root,'frontend/dist')): - initialize_rootdir(Globals.root,opt.yes_to_all) + initialize_rootdir(Globals.root,(not opt.interactive) or opt.yes_to_all) print(f'(Initializing with runtime root {Globals.root})\n') diff --git a/setup.py b/setup.py index bf33b31345..9916ace17d 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def _get_requirements(path): try: with open(path) as f: packages = f.read().splitlines() - except OSError as ex: - raise RuntimeError('Cannot open file with requirements: %s', repr(ex)) + except (IOError, OSError) as ex: + raise RuntimeError("Can't open file with requirements: %s", repr(ex)) # Drop option lines packages = [package for package in packages if not re.match(r'^--', package)] @@ -74,4 +74,6 @@ setup( 'Topic :: Scientific/Engineering :: Artificial Intelligence', 'Topic :: Scientific/Engineering :: Image Processing', ], + scripts = ['scripts/invoke.py','scripts/configure_invokeai.py', 'scripts/preload_models.py', 'scripts/sd-metadata.py'], + data_files=[('frontend',frontend_files)], )