diff --git a/.github/workflows/test-invoke-conda.yml b/.github/workflows/test-invoke-conda.yml
index 1bed3433e9..65e3e4e1e2 100644
--- a/.github/workflows/test-invoke-conda.yml
+++ b/.github/workflows/test-invoke-conda.yml
@@ -19,9 +19,11 @@ jobs:
           - os: ubuntu-latest
             environment-file: environment.yml
             default-shell: bash -l {0}
+            stable-diffusion-model: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt
           - os: macos-latest
             environment-file: environment-mac.yml
             default-shell: bash -l {0}
+            stable-diffusion-model: https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt
     name: Test invoke.py on ${{ matrix.os }} with conda
     runs-on: ${{ matrix.os }}
     defaults:
@@ -57,7 +59,7 @@ jobs:
         id: cache-sd-v1-4
         uses: actions/cache@v3
         env:
-          cache-name: cache-sd-v1-4
+          cache-name: cache-sd-${{ matrix.stable-diffusion-model }}
         with:
           path: models/ldm/stable-diffusion-v1/model.ckpt
           key: ${{ env.cache-name }}
@@ -71,7 +73,7 @@ jobs:
           [[ -r models/ldm/stable-diffusion-v1/model.ckpt ]] \
             || curl --user "${{ secrets.HUGGINGFACE_TOKEN }}" \
               -o models/ldm/stable-diffusion-v1/model.ckpt \
-              -O -L https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt
+              -O -L ${{ matrix.stable-diffusion-model }}
 
       - name: Activate Conda Env
         uses: conda-incubator/setup-miniconda@v2