Added linux to the workflows (#463)

* Added linux to the workflows

- rename workflow files

Signed-off-by: Ben Alkov <ben.alkov@gmail.com>

* fixes: run on merge to 'main', 'dev'; 

- reduce dev merge test cases to 1 (1 takes 11 minutes 😯)
- fix model cache name

Signed-off-by: Ben Alkov <ben.alkov@gmail.com>

* add test prompts to workflows

Signed-off-by: Ben Alkov <ben.alkov@gmail.com>

Signed-off-by: Ben Alkov <ben.alkov@gmail.com>
Co-authored-by: James Reynolds <magnsuviri@me.com>
Co-authored-by: Ben Alkov <ben.alkov@gmail.com>
Co-authored-by: Lincoln Stein <lincoln.stein@gmail.com>

.github/workflows/cache-model.yml

@@ -1,64 +0,0 @@
-name: Cache Model
-on:
-  workflow_dispatch
-jobs:
-  build:
-    strategy:
-      matrix:
-        os: [ macos-12 ]
-    name: Create Caches using ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Cache model
-        id: cache-sd-v1-4
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-sd-v1-4
-        with:
-          path: models/ldm/stable-diffusion-v1/model.ckpt
-          key: ${{ env.cache-name }}
-          restore-keys: |
-            ${{ env.cache-name }}
-      - name: Download Stable Diffusion v1.4 model
-        if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }}
-        continue-on-error: true
-        run: |
-          if [ ! -e models/ldm/stable-diffusion-v1 ]; then
-              mkdir -p models/ldm/stable-diffusion-v1
-          fi
-          if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then
-            curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }}
-          fi
-# Uncomment this when we no longer make changes to environment-mac.yaml
-#       - name: Cache environment
-#         id: cache-conda-env-ldm
-#         uses: actions/cache@v3
-#         env:
-#           cache-name: cache-conda-env-ldm
-#         with:
-#           path: ~/.conda/envs/ldm
-#           key: ${{ env.cache-name }}
-#           restore-keys: |
-#             ${{ env.cache-name }}
-      - name: Install dependencies
-#         if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }}
-        run: |
-          conda env create -f environment-mac.yaml
-      - name: Cache hugginface and torch models
-        id: cache-hugginface-torch
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-hugginface-torch
-        with:
-          path: ~/.cache
-          key: ${{ env.cache-name }}
-          restore-keys: |
-            ${{ env.cache-name }}
-      - name: Download Huggingface and Torch models
-        if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }}
-        continue-on-error: true
-        run: |
-          export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python
-          $PYTHON_BIN scripts/preload_models.py

.github/workflows/create-caches.yml

@@ -0,0 +1,70 @@
+name: Create Caches
+on:
+  workflow_dispatch
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-12 ]
+    name: Create Caches on ${{ matrix.os }} conda
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set platform variables
+        id: vars
+        run: |
+            if [ "$RUNNER_OS" = "macOS" ]; then
+                echo "::set-output name=ENV_FILE::environment-mac.yaml"
+                echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python"
+            elif [ "$RUNNER_OS" = "Linux" ]; then
+                echo "::set-output name=ENV_FILE::environment.yaml"
+                echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python"
+            fi
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Use Cached Stable Diffusion v1.4 Model
+        id: cache-sd-v1-4
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-sd-v1-4
+        with:
+          path: models/ldm/stable-diffusion-v1/model.ckpt
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
+      - name: Download Stable Diffusion v1.4 Model
+        if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }}
+        run: |
+          if [ ! -e models/ldm/stable-diffusion-v1 ]; then
+              mkdir -p models/ldm/stable-diffusion-v1
+          fi
+          if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then
+            curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }}
+          fi
+      - name: Use Cached Dependencies
+        id: cache-conda-env-ldm
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-conda-env-ldm
+        with:
+          path: ~/.conda/envs/ldm
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }}
+      - name: Install Dependencies
+        if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }}
+        run: |
+          conda env create -f ${{ steps.vars.outputs.ENV_FILE }}
+      - name: Use Cached Huggingface and Torch models
+        id: cache-huggingface-torch
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-huggingface-torch
+        with:
+          path: ~/.cache
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }}
+      - name: Download Huggingface and Torch models
+        if: ${{ steps.cache-huggingface-torch.outputs.cache-hit != 'true' }}
+        run: |
+          ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py

.github/workflows/macos12-miniconda.yml

@@ -1,80 +0,0 @@
-name: Build
-on: 
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-jobs:
-  build:
-    strategy:
-      matrix:
-        os: [ macos-12 ]
-    name: Build on ${{ matrix.os }} miniconda
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Cache model
-        id: cache-sd-v1-4
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-sd-v1-4
-        with:
-          path: models/ldm/stable-diffusion-v1/model.ckpt
-          key: ${{ env.cache-name }}
-          restore-keys: |
-            ${{ env.cache-name }}
-      - name: Download Stable Diffusion v1.4 model
-        if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }}
-        continue-on-error: true
-        run: |
-          if [ ! -e models/ldm/stable-diffusion-v1 ]; then
-              mkdir -p models/ldm/stable-diffusion-v1
-          fi
-          if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then
-            curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }}
-          fi
-# Uncomment this when we no longer make changes to environment-mac.yaml
-#       - name: Cache environment
-#         id: cache-conda-env-ldm
-#         uses: actions/cache@v3
-#         env:
-#           cache-name: cache-conda-env-ldm
-#         with:
-#           path: ~/.conda/envs/ldm
-#           key: ${{ env.cache-name }}
-#           restore-keys: |
-#             ${{ env.cache-name }}
-      - name: Install dependencies
-#         if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }}
-        run: |
-          conda env create -f environment-mac.yaml
-      - name: Cache hugginface and torch models
-        id: cache-hugginface-torch
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-hugginface-torch
-        with:
-          path: ~/.cache
-          key: ${{ env.cache-name }}
-          restore-keys: |
-            ${{ env.cache-name }}
-      - name: Download Huggingface and Torch models
-        if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }}
-        continue-on-error: true
-        run: |
-          export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python
-          $PYTHON_BIN scripts/preload_models.py
-      - name: Run the tests
-        run: |
-          # Note, can't "activate" via automation, and activation is just env vars and path
-          export PYTHON_BIN=/usr/local/miniconda/envs/ldm/bin/python
-          export PYTORCH_ENABLE_MPS_FALLBACK=1
-          $PYTHON_BIN scripts/preload_models.py
-          mkdir -p outputs/img-samples
-          time $PYTHON_BIN scripts/dream.py --from_file tests/prompts.txt </dev/null 2> outputs/img-samples/err.log > outputs/img-samples/out.log
-      - name: Archive results
-        uses: actions/upload-artifact@v3
-        with:
-          name: results
-          path: outputs/img-samples

.github/workflows/test-dream-conda.yml

@@ -0,0 +1,97 @@
+name: Test Dream with Conda
+on:
+  push:
+    branches:
+      - 'main'
+      - 'development'
+jobs:
+  os_matrix:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-12 ]
+    name: Test dream.py on ${{ matrix.os }} with conda
+    runs-on: ${{ matrix.os }}
+    steps:
+      - run: |
+          echo The PR was merged
+      - name: Set platform variables
+        id: vars
+        run: |
+          # Note, can't "activate" via github action; specifying the env's python has the same effect
+          if [ "$RUNNER_OS" = "macOS" ]; then
+              echo "::set-output name=ENV_FILE::environment-mac.yaml"
+              echo "::set-output name=PYTHON_BIN::/usr/local/miniconda/envs/ldm/bin/python"
+          elif [ "$RUNNER_OS" = "Linux" ]; then
+              echo "::set-output name=ENV_FILE::environment.yaml"
+              echo "::set-output name=PYTHON_BIN::/usr/share/miniconda/envs/ldm/bin/python"
+          fi
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Use Cached Stable Diffusion v1.4 Model
+        id: cache-sd-v1-4
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-sd-v1-4
+        with:
+          path: models/ldm/stable-diffusion-v1/model.ckpt
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
+      - name: Download Stable Diffusion v1.4 Model
+        if: ${{ steps.cache-sd-v1-4.outputs.cache-hit != 'true' }}
+        run: |
+          if [ ! -e models/ldm/stable-diffusion-v1 ]; then
+              mkdir -p models/ldm/stable-diffusion-v1
+          fi
+          if [ ! -e models/ldm/stable-diffusion-v1/model.ckpt ]; then
+            curl -o models/ldm/stable-diffusion-v1/model.ckpt ${{ secrets.SD_V1_4_URL }}
+          fi
+      - name: Use Cached Dependencies
+        id: cache-conda-env-ldm
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-conda-env-ldm
+        with:
+          path: ~/.conda/envs/ldm
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(steps.vars.outputs.ENV_FILE) }}
+      - name: Install Dependencies
+        if: ${{ steps.cache-conda-env-ldm.outputs.cache-hit != 'true' }}
+        run: |
+          conda env create -f ${{ steps.vars.outputs.ENV_FILE }}
+      - name: Use Cached Huggingface and Torch models
+        id: cache-hugginface-torch
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-hugginface-torch
+        with:
+          path: ~/.cache
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}-${{ hashFiles('scripts/preload_models.py') }}
+      - name: Download Huggingface and Torch models
+        if: ${{ steps.cache-hugginface-torch.outputs.cache-hit != 'true' }}
+        run: |
+          ${{ steps.vars.outputs.PYTHON_BIN }} scripts/preload_models.py
+#       - name: Run tmate
+#         uses: mxschmitt/action-tmate@v3
+#         timeout-minutes: 30
+      - name: Run the tests
+        run: |
+          # Note, can't "activate" via github action; specifying the env's python has the same effect
+          if [ $(uname) = "Darwin" ]; then
+              export PYTORCH_ENABLE_MPS_FALLBACK=1
+          fi
+          # Utterly hacky, but I don't know how else to do this
+          if [[ ${{ github.ref }} == 'refs/heads/master' ]]; then
+            time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/preflight_prompts.txt --full_precision
+          elif [[ ${{ github.ref }} == 'refs/heads/development' ]]; then
+            time ${{ steps.vars.outputs.PYTHON_BIN }} scripts/dream.py --from_file tests/dev_prompts.txt --full_precision
+          fi
+          mkdir -p outputs/img-samples
+      - name: Archive results
+        uses: actions/upload-artifact@v3
+        with:
+          name: results
+          path: outputs/img-samples

tests/dev_prompts.txt

@@ -0,0 +1 @@
+banana sushi -Ak_lms -S42

tests/preflight_prompts.txt

@@ -0,0 +1,9 @@
+banana sushi -Ak_lms -S42
+banana sushi -Addim -S42
+banana sushi -Ak_lms -W640 -H480 -S42
+banana sushi -Ak_lms -S42 -G1 -U 2 0.5
+banana sushi -Ak_lms -S42 -v0.2 -n3
+banana sushi -Ak_lms -S42 -V1349749425:0.1,4145759947:0.1
+snake -I outputs/preflight/000006.4145759947.png -S42
+snake -I outputs/preflight/000006.4145759947.png -S42 -W640 -H640 --fit
+strawberry sushi -I./image-and-mask.png -S42 -f0.9 -s100 -C15

tests/prompts.txt

@@ -1 +0,0 @@
-test trending on artstation -s 1 -S 1