From 37e2418ee0432e64820d3e27cdb81e02a6bf0a94 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Fri, 16 Sep 2022 15:46:57 -0600 Subject: [PATCH] Added linux to the workflows (#463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added linux to the workflows - rename workflow files Signed-off-by: Ben Alkov * fixes: run on merge to 'main', 'dev'; - reduce dev merge test cases to 1 (1 takes 11 minutes 😯) - fix model cache name Signed-off-by: Ben Alkov * add test prompts to workflows Signed-off-by: Ben Alkov Signed-off-by: Ben Alkov Co-authored-by: James Reynolds Co-authored-by: Ben Alkov Co-authored-by: Lincoln Stein --- .github/workflows/cache-model.yml | 64 ---------------- .github/workflows/create-caches.yml | 70 ++++++++++++++++++ .github/workflows/macos12-miniconda.yml | 80 -------------------- .github/workflows/test-dream-conda.yml | 97 +++++++++++++++++++++++++ tests/dev_prompts.txt | 1 + tests/preflight_prompts.txt | 9 +++ tests/prompts.txt | 1 - 7 files changed, 177 insertions(+), 145 deletions(-) delete mode 100644 .github/workflows/cache-model.yml create mode 100644 .github/workflows/create-caches.yml delete mode 100644 .github/workflows/macos12-miniconda.yml create mode 100644 .github/workflows/test-dream-conda.yml create mode 100644 tests/dev_prompts.txt create mode 100644 tests/preflight_prompts.txt delete mode 100644 tests/prompts.txt diff --git a/.github/workflows/cache-model.yml b/.github/workflows/cache-model.yml deleted file mode 100644 index 2682943eef..0000000000 --- a/.github/workflows/cache-model.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Cache Model -on: - workflow_dispatch -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Create Caches using ${{ matrix.os }} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py \ No newline at end of file diff --git a/.github/workflows/create-caches.yml b/.github/workflows/create-caches.yml new file mode 100644 index 0000000000..951718af1b --- /dev/null +++ b/.github/workflows/create-caches.yml @@ -0,0 +1,70 @@ +name: Create Caches +on: + workflow_dispatch +jobs: + build: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Create Caches on ${{ matrix.os }} conda + runs-on: ${{ matrix.os }} + steps: + - name: Set platform variables + id: vars + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-huggingface-torch + uses: actions/cache@v3 + env: + cache-name: cache-huggingface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-huggingface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py diff --git a/.github/workflows/macos12-miniconda.yml b/.github/workflows/macos12-miniconda.yml deleted file mode 100644 index 18f21277c0..0000000000 --- a/.github/workflows/macos12-miniconda.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] -jobs: - build: - strategy: - matrix: - os: [ macos-12 ] - name: Build on ${{ matrix.os }} miniconda - runs-on: ${{ matrix.os }} - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Cache model - id: cache-sd-v1-4 - uses: actions/cache@v3 - env: - cache-name: cache-sd-v1-4 - with: - path: models/ldm/stable-diffusion-v1/model.ckpt - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Stable Diffusion v1.4 model - if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - if [ ! -e models/ldm/stable-diffusion-v1 ]; then - mkdir -p models/ldm/stable-diffusion-v1 - fi - if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then - curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} - fi -# Uncomment this when we no longer make changes to environment-mac.yaml -# - name: Cache environment -# id: cache-conda-env-ldm -# uses: actions/cache@v3 -# env: -# cache-name: cache-conda-env-ldm -# with: -# path: ~/.conda/envs/ldm -# key: ${{ env.cache-name }} -# restore-keys: | -# ${{ env.cache-name }} - - name: Install dependencies -# if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} - run: | - conda env create -f environment-mac.yaml - - name: Cache hugginface and torch models - id: cache-hugginface-torch - uses: actions/cache@v3 - env: - cache-name: cache-hugginface-torch - with: - path: ~/.cache - key: ${{ env.cache-name }} - restore-keys: | - ${{ env.cache-name }} - - name: Download Huggingface and Torch models - if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} - continue-on-error: true - run: | - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - $PYTHON_BIN scripts/preload_models.py - - name: Run the tests - run: | - # Note, can't "activate" via automation, and activation is just env vars and path - export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python - export PYTORCH_ENABLE_MPS_FALLBACK=1 - $PYTHON_BIN scripts/preload_models.py - mkdir -p outputs/img-samples - time $PYTHON_BIN scripts/dream.py --from_file tests/prompts.txt outputs/img-samples/err.log > outputs/img-samples/out.log - - name: Archive results - uses: actions/upload-artifact@v3 - with: - name: results - path: outputs/img-samples \ No newline at end of file diff --git a/.github/workflows/test-dream-conda.yml b/.github/workflows/test-dream-conda.yml new file mode 100644 index 0000000000..3bd9b24582 --- /dev/null +++ b/.github/workflows/test-dream-conda.yml @@ -0,0 +1,97 @@ +name: Test Dream with Conda +on: + push: + branches: + - 'main' + - 'development' +jobs: + os_matrix: + strategy: + matrix: + os: [ ubuntu-latest, macos-12 ] + name: Test dream.py on ${{ matrix.os }} with conda + runs-on: ${{ matrix.os }} + steps: + - run: | + echo The PR was merged + - name: Set platform variables + id: vars + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ "$RUNNER_OS" = "macOS" ]; then + echo "::set-output name=ENV_FILE::environment-mac.yaml" + echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python" + elif [ "$RUNNER_OS" = "Linux" ]; then + echo "::set-output name=ENV_FILE::environment.yaml" + echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python" + fi + - name: Checkout sources + uses: actions/checkout@v3 + - name: Use Cached Stable Diffusion v1.4 Model + id: cache-sd-v1-4 + uses: actions/cache@v3 + env: + cache-name: cache-sd-v1-4 + with: + path: models/ldm/stable-diffusion-v1/model.ckpt + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }} + - name: Download Stable Diffusion v1.4 Model + if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }} + run: | + if [ ! -e models/ldm/stable-diffusion-v1 ]; then + mkdir -p models/ldm/stable-diffusion-v1 + fi + if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then + curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }} + fi + - name: Use Cached Dependencies + id: cache-conda-env-ldm + uses: actions/cache@v3 + env: + cache-name: cache-conda-env-ldm + with: + path: ~/.conda/envs/ldm + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }} + - name: Install Dependencies + if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }} + run: | + conda env create -f ${{ steps.vars.outputs.ENV_FILE }} + - name: Use Cached Huggingface and Torch models + id: cache-hugginface-torch + uses: actions/cache@v3 + env: + cache-name: cache-hugginface-torch + with: + path: ~/.cache + key: ${{ env.cache-name }} + restore-keys: | + ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }} + - name: Download Huggingface and Torch models + if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }} + run: | + ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py +# - name: Run tmate +# uses: mxschmitt/action-tmate@v3 +# timeout-minutes: 30 + - name: Run the tests + run: | + # Note, can't "activate" via github action; specifying the env's python has the same effect + if [ $(uname) = "Darwin" ]; then + export PYTORCH_ENABLE_MPS_FALLBACK=1 + fi + # Utterly hacky, but I don't know how else to do this + if [[ ${{ github.ref }} == 'refs/heads/master' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/preflight_prompts.txt --full_precision + elif [[ ${{ github.ref }} == 'refs/heads/development' ]]; then + time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/dev_prompts.txt --full_precision + fi + mkdir -p outputs/img-samples + - name: Archive results + uses: actions/upload-artifact@v3 + with: + name: results + path: outputs/img-samples diff --git a/tests/dev_prompts.txt b/tests/dev_prompts.txt new file mode 100644 index 0000000000..9ebca4e9f7 --- /dev/null +++ b/tests/dev_prompts.txt @@ -0,0 +1 @@ +banana sushi -Ak_lms -S42 diff --git a/tests/preflight_prompts.txt b/tests/preflight_prompts.txt new file mode 100644 index 0000000000..5c5b8233a1 --- /dev/null +++ b/tests/preflight_prompts.txt @@ -0,0 +1,9 @@ +banana sushi -Ak_lms -S42 +banana sushi -Addim -S42 +banana sushi -Ak_lms -W640 -H480 -S42 +banana sushi -Ak_lms -S42 -G1 -U 2 0.5 +banana sushi -Ak_lms -S42 -v0.2 -n3 +banana sushi -Ak_lms -S42 -V1349749425:0.1,4145759947:0.1 +snake -I outputs/preflight/000006.4145759947.png -S42 +snake -I outputs/preflight/000006.4145759947.png -S42 -W640 -H640 --fit +strawberry sushi -I./image-and-mask.png -S42 -f0.9 -s100 -C15 \ No newline at end of file diff --git a/tests/prompts.txt b/tests/prompts.txt deleted file mode 100644 index 955220a5e6..0000000000 --- a/tests/prompts.txt +++ /dev/null @@ -1 +0,0 @@ -test trending on artstation -s 1 -S 1