From eef3373799201f354dbf5b76f1c2738db2946bd6 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Sat, 2 Mar 2024 21:37:49 +1100 Subject: [PATCH] ci: fix workflows Do not split up "on change" and "do the thing". Less convoluted, no catch-22 with required checks for PRs. --- .../actions/install-frontend-deps/action.yml | 14 +-- .../actions/install-python-deps/action.yml | 11 -- .github/workflows/check-frontend.yml | 43 ------- .github/workflows/check-python.yml | 33 ------ .github/workflows/frontend-checks.yml | 62 +++++++++++ .github/workflows/frontend-tests.yml | 42 +++++++ .github/workflows/mkdocs-material.yml | 23 +++- .../workflows/on-change-check-frontend.yml | 39 ------- .github/workflows/on-change-check-python.yml | 42 ------- .github/workflows/on-change-pytest.yml | 42 ------- .github/workflows/python-checks.yml | 58 ++++++++++ .../{check-pytest.yml => python-test.yml} | 41 ++++--- .github/workflows/release.yml | 105 ++++++++++++------ 13 files changed, 287 insertions(+), 268 deletions(-) delete mode 100644 .github/actions/install-python-deps/action.yml delete mode 100644 .github/workflows/check-frontend.yml delete mode 100644 .github/workflows/check-python.yml create mode 100644 .github/workflows/frontend-checks.yml create mode 100644 .github/workflows/frontend-tests.yml delete mode 100644 .github/workflows/on-change-check-frontend.yml delete mode 100644 .github/workflows/on-change-check-python.yml delete mode 100644 .github/workflows/on-change-pytest.yml create mode 100644 .github/workflows/python-checks.yml rename .github/workflows/{check-pytest.yml => python-test.yml} (63%) diff --git a/.github/actions/install-frontend-deps/action.yml b/.github/actions/install-frontend-deps/action.yml index b9d910ca99..32b4987249 100644 --- a/.github/actions/install-frontend-deps/action.yml +++ b/.github/actions/install-frontend-deps/action.yml @@ -1,33 +1,33 @@ -name: Install frontend dependencies +name: install frontend dependencies description: Installs frontend dependencies with pnpm, with caching runs: using: 'composite' steps: - - name: Setup Node 18 + - name: setup node 18 uses: actions/setup-node@v4 with: node-version: '18' - - name: Setup pnpm + - name: setup pnpm uses: pnpm/action-setup@v2 with: version: 8 run_install: false - - name: Get pnpm store directory + - name: get pnpm store directory shell: bash run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - uses: actions/cache@v3 - name: Setup pnpm cache + - name: setup cache + uses: actions/cache@v4 with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install frontend dependencies + - name: install frontend dependencies run: pnpm install --prefer-frozen-lockfile shell: bash working-directory: invokeai/frontend/web diff --git a/.github/actions/install-python-deps/action.yml b/.github/actions/install-python-deps/action.yml deleted file mode 100644 index 4c0d351899..0000000000 --- a/.github/actions/install-python-deps/action.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Install python dependencies -description: Install python dependencies with pip, with caching -runs: - using: 'composite' - steps: - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - cache: pip - cache-dependency-path: pyproject.toml diff --git a/.github/workflows/check-frontend.yml b/.github/workflows/check-frontend.yml deleted file mode 100644 index 8134926556..0000000000 --- a/.github/workflows/check-frontend.yml +++ /dev/null @@ -1,43 +0,0 @@ -# This workflow runs the frontend code quality checks. -# -# It may be triggered via dispatch, or by another workflow. - -name: 'Check: frontend' - -on: - workflow_dispatch: - workflow_call: - -defaults: - run: - working-directory: invokeai/frontend/web - -jobs: - check-frontend: - runs-on: ubuntu-latest - timeout-minutes: 10 # expected run time: <2 min - steps: - - uses: actions/checkout@v4 - - - name: Set up frontend - uses: ./.github/actions/install-frontend-deps - - - name: Run tsc check - run: 'pnpm run lint:tsc' - shell: bash - - - name: Run dpdm check - run: 'pnpm run lint:dpdm' - shell: bash - - - name: Run eslint check - run: 'pnpm run lint:eslint' - shell: bash - - - name: Run prettier check - run: 'pnpm run lint:prettier' - shell: bash - - - name: Run knip check - run: 'pnpm run lint:knip' - shell: bash diff --git a/.github/workflows/check-python.yml b/.github/workflows/check-python.yml deleted file mode 100644 index 63a6c46b0a..0000000000 --- a/.github/workflows/check-python.yml +++ /dev/null @@ -1,33 +0,0 @@ -# This workflow runs the python code quality checks. -# -# It may be triggered via dispatch, or by another workflow. -# -# TODO: Add mypy or pyright to the checks. - -name: 'Check: python' - -on: - workflow_dispatch: - workflow_call: - -jobs: - check-backend: - runs-on: ubuntu-latest - timeout-minutes: 5 # expected run time: <1 min - steps: - - uses: actions/checkout@v4 - - - name: Install python dependencies - uses: ./.github/actions/install-python-deps - - - name: Install ruff - run: pip install ruff - shell: bash - - - name: Ruff check - run: ruff check --output-format=github . - shell: bash - - - name: Ruff format - run: ruff format --check . - shell: bash diff --git a/.github/workflows/frontend-checks.yml b/.github/workflows/frontend-checks.yml new file mode 100644 index 0000000000..86f15fd562 --- /dev/null +++ b/.github/workflows/frontend-checks.yml @@ -0,0 +1,62 @@ +name: 'frontend checks' + +on: + push: + branches: + - 'main' + pull_request: + types: + - 'ready_for_review' + - 'opened' + - 'synchronize' + merge_group: + workflow_dispatch: + workflow_call: + +defaults: + run: + working-directory: invokeai/frontend/web + +jobs: + check-frontend: + runs-on: ubuntu-latest + timeout-minutes: 10 # expected run time: <2 min + steps: + - uses: actions/checkout@v4 + + - name: check for changed frontend files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files_yaml: | + frontend: + - 'invokeai/frontend/web/**' + + - name: install dependencies + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + uses: ./.github/actions/install-frontend-deps + + - name: tsc + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm lint:tsc' + shell: bash + + - name: dpdm + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm lint:dpdm' + shell: bash + + - name: eslint + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm lint:eslint' + shell: bash + + - name: prettier + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm lint:prettier' + shell: bash + + - name: knip + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm lint:knip' + shell: bash diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml new file mode 100644 index 0000000000..9ac74f16c0 --- /dev/null +++ b/.github/workflows/frontend-tests.yml @@ -0,0 +1,42 @@ +name: 'frontend tests' + +on: + push: + branches: + - 'main' + pull_request: + types: + - 'ready_for_review' + - 'opened' + - 'synchronize' + merge_group: + workflow_dispatch: + workflow_call: + +defaults: + run: + working-directory: invokeai/frontend/web + +jobs: + check-frontend: + runs-on: ubuntu-latest + timeout-minutes: 10 # expected run time: <2 min + steps: + - uses: actions/checkout@v4 + + - name: check for changed frontend files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files_yaml: | + frontend: + - 'invokeai/frontend/web/**' + + - name: install dependencies + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + uses: ./.github/actions/install-frontend-deps + + - name: vitest + if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' }} + run: 'pnpm test:no-watch' + shell: bash diff --git a/.github/workflows/mkdocs-material.yml b/.github/workflows/mkdocs-material.yml index cbcfbf0835..419d87f37b 100644 --- a/.github/workflows/mkdocs-material.yml +++ b/.github/workflows/mkdocs-material.yml @@ -21,18 +21,29 @@ jobs: SITE_URL: 'https://${{ github.repository_owner }}.github.io/InvokeAI' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: checkout + uses: actions/checkout@v4 + + - name: setup python + uses: actions/setup-python@v5 with: python-version: '3.10' cache: pip cache-dependency-path: pyproject.toml - - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v4 + + - name: set cache id + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: use cache + uses: actions/cache@v4 with: key: mkdocs-material-${{ env.cache_id }} path: .cache restore-keys: | mkdocs-material- - - run: python -m pip install ".[docs]" - - run: mkdocs gh-deploy --force + + - name: install dependencies + run: python -m pip install ".[docs]" + + - name: build & deploy + run: mkdocs gh-deploy --force diff --git a/.github/workflows/on-change-check-frontend.yml b/.github/workflows/on-change-check-frontend.yml deleted file mode 100644 index 5e8704ad71..0000000000 --- a/.github/workflows/on-change-check-frontend.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow runs of `check-frontend.yml` on push or pull request. -# -# The actual checks are in a separate workflow to support simpler workflow -# composition without awkward or complicated conditionals. - -name: 'On change: run check-frontend' - -on: - push: - branches: - - 'main' - pull_request: - types: - - 'ready_for_review' - - 'opened' - - 'synchronize' - merge_group: - -jobs: - check-changed-frontend-files: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - outputs: - frontend_any_changed: ${{ steps.changed-files.outputs.frontend_any_changed }} - steps: - - uses: actions/checkout@v4 - - - name: Check for changed frontend files - id: changed-files - uses: tj-actions/changed-files@v41 - with: - files_yaml: | - frontend: - - 'invokeai/frontend/web/**' - - run-check-frontend: - needs: check-changed-frontend-files - if: ${{ needs.check-changed-frontend-files.outputs.frontend_any_changed == 'true' }} - uses: ./.github/workflows/check-frontend.yml diff --git a/.github/workflows/on-change-check-python.yml b/.github/workflows/on-change-check-python.yml deleted file mode 100644 index e73198b3fa..0000000000 --- a/.github/workflows/on-change-check-python.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow runs of `check-python.yml` on push or pull request. -# -# The actual checks are in a separate workflow to support simpler workflow -# composition without awkward or complicated conditionals. - -name: 'On change: run check-python' - -on: - push: - branches: - - 'main' - pull_request: - types: - - 'ready_for_review' - - 'opened' - - 'synchronize' - merge_group: - -jobs: - check-changed-python-files: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - outputs: - python_any_changed: ${{ steps.changed-files.outputs.python_any_changed }} - steps: - - uses: actions/checkout@v4 - - - name: Check for changed python files - id: changed-files - uses: tj-actions/changed-files@v41 - with: - files_yaml: | - python: - - 'pyproject.toml' - - 'invokeai/**' - - '!invokeai/frontend/web/**' - - 'tests/**' - - run-check-python: - needs: check-changed-python-files - if: ${{ needs.check-changed-python-files.outputs.python_any_changed == 'true' }} - uses: ./.github/workflows/check-python.yml diff --git a/.github/workflows/on-change-pytest.yml b/.github/workflows/on-change-pytest.yml deleted file mode 100644 index 0c174098bb..0000000000 --- a/.github/workflows/on-change-pytest.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow runs of `check-pytest.yml` on push or pull request. -# -# The actual checks are in a separate workflow to support simpler workflow -# composition without awkward or complicated conditionals. - -name: 'On change: run pytest' - -on: - push: - branches: - - 'main' - pull_request: - types: - - 'ready_for_review' - - 'opened' - - 'synchronize' - merge_group: - -jobs: - check-changed-python-files: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - outputs: - python_any_changed: ${{ steps.changed-files.outputs.python_any_changed }} - steps: - - uses: actions/checkout@v4 - - - name: Check for changed python files - id: changed-files - uses: tj-actions/changed-files@v41 - with: - files_yaml: | - python: - - 'pyproject.toml' - - 'invokeai/**' - - '!invokeai/frontend/web/**' - - 'tests/**' - - run-pytest: - needs: check-changed-python-files - if: ${{ needs.check-changed-python-files.outputs.python_any_changed == 'true' }} - uses: ./.github/workflows/check-pytest.yml diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml new file mode 100644 index 0000000000..2496942148 --- /dev/null +++ b/.github/workflows/python-checks.yml @@ -0,0 +1,58 @@ +# TODO: Add mypy or pyright to the checks. + +name: 'python checks' + +on: + push: + branches: + - 'main' + pull_request: + types: + - 'ready_for_review' + - 'opened' + - 'synchronize' + merge_group: + workflow_dispatch: + workflow_call: + +jobs: + check-backend: + runs-on: ubuntu-latest + timeout-minutes: 5 # expected run time: <1 min + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: check for changed python files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files_yaml: | + python: + - 'pyproject.toml' + - 'invokeai/**' + - '!invokeai/frontend/web/**' + - 'tests/**' + + - name: setup python + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: pyproject.toml + + - name: install ruff + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + run: pip install ruff + shell: bash + + - name: ruff check + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + run: ruff check --output-format=github . + shell: bash + + - name: ruff format + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + run: ruff format --check . + shell: bash diff --git a/.github/workflows/check-pytest.yml b/.github/workflows/python-test.yml similarity index 63% rename from .github/workflows/check-pytest.yml rename to .github/workflows/python-test.yml index aedc0e59c3..7f8ec5af5a 100644 --- a/.github/workflows/check-pytest.yml +++ b/.github/workflows/python-test.yml @@ -1,10 +1,15 @@ -# This workflow runs pytest on the codebase in a matrix of platforms. -# -# It may be triggered via dispatch, or by another workflow. - -name: 'Check: pytest' +name: 'python tests' on: + push: + branches: + - 'main' + pull_request: + types: + - 'ready_for_review' + - 'opened' + - 'synchronize' + merge_group: workflow_dispatch: workflow_call: @@ -44,29 +49,39 @@ jobs: github-env: $env:GITHUB_ENV name: ${{ matrix.pytorch }} on ${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 30 # expected run time: <10 min, depending on platform + timeout-minutes: 15 # expected run time: 2-6 min, depending on platform env: PIP_USE_PEP517: '1' steps: - - uses: actions/checkout@v4 + - name: checkout + uses: actions/checkout@v4 - - name: set test prompt to main branch validation - run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }} + - name: check for changed python files + id: changed-files + uses: tj-actions/changed-files@v42 + with: + files_yaml: | + python: + - 'pyproject.toml' + - 'invokeai/**' + - '!invokeai/frontend/web/**' + - 'tests/**' - name: setup python + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml - - name: install invokeai + - name: install dependencies + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} env: PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }} run: > - pip3 install - --editable=".[test]" + pip3 install --editable=".[test]" - name: run pytest - id: run-pytest + if: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} run: pytest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f9ca098d5..bce20d15af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: build & release on: push: @@ -7,60 +7,85 @@ on: workflow_dispatch: inputs: skip_code_checks: - description: 'Skip code checks' + description: 'Skip code checks (disables publish)' required: true default: true type: boolean jobs: - check-version: + frontend-checks: + if: github.event.inputs.skip_code_checks != 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: run-frontend-checks + uses: ./.github/workflows/frontend-checks.yml - - uses: samuelcolvin/check-python-version@v4 + frontend-tests: + if: github.event.inputs.skip_code_checks != 'true' + runs-on: ubuntu-latest + steps: + - name: run-frontend-tests + uses: ./.github/workflows/frontend-tests.yml + + python-checks: + if: github.event.inputs.skip_code_checks != 'true' + runs-on: ubuntu-latest + steps: + - name: run-python-checks + uses: ./.github/workflows/python-checks.yml + + python-tests: + if: github.event.inputs.skip_code_checks != 'true' + runs-on: ubuntu-latest + steps: + - name: run-python-tests + uses: ./.github/workflows/python-tests.yml + + check-version: + if: github.event.inputs.skip_code_checks != 'true' + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: check python version + uses: samuelcolvin/check-python-version@v4 id: check-python-version with: version_file_path: invokeai/version/invokeai_version.py - check-frontend: - if: github.event.inputs.skip_code_checks != 'true' - uses: ./.github/workflows/check-frontend.yml - - check-python: - if: github.event.inputs.skip_code_checks != 'true' - uses: ./.github/workflows/check-python.yml - - check-pytest: - if: github.event.inputs.skip_code_checks != 'true' - uses: ./.github/workflows/check-pytest.yml - build: runs-on: ubuntu-latest + timeout-minutes: 15 # expected run time: <10 min steps: - - uses: actions/checkout@v4 + - name: checkout + uses: actions/checkout@v4 - - name: Install python dependencies - uses: ./.github/actions/install-python-deps + - name: setup python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: pyproject.toml - - name: Install pypa/build + - name: install pypa/build run: pip install --upgrade build - - name: Setup frontend + - name: setup frontend uses: ./.github/actions/install-frontend-deps - - name: Run create_installer.sh + - name: create installer id: create_installer - run: ./create_installer.sh --skip_frontend_checks + run: ./create_installer.sh working-directory: installer - - name: Upload python distribution artifact + - name: upload python distribution artifact uses: actions/upload-artifact@v4 with: name: dist path: ${{ steps.create_installer.outputs.DIST_PATH }} - - name: Upload installer artifact + - name: upload installer artifact uses: actions/upload-artifact@v4 with: name: ${{ steps.create_installer.outputs.INSTALLER_FILENAME }} @@ -68,36 +93,52 @@ jobs: publish-testpypi: runs-on: ubuntu-latest - needs: [check-version, check-frontend, check-python, check-pytest, build] + needs: + [ + check-version, + frontend-checks, + frontend-tests, + python-checks, + python-tests, + build, + ] if: github.event_name != 'workflow_dispatch' environment: name: testpypi url: https://test.pypi.org/p/invokeai steps: - - name: Download distribution from build job + - name: download distribution from build job uses: actions/download-artifact@v4 with: name: dist path: dist/ - - name: Publish distribution to TestPyPI + - name: publish distribution to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ publish-pypi: runs-on: ubuntu-latest - needs: [check-version, check-frontend, check-python, check-pytest, build] + needs: + [ + check-version, + frontend-checks, + frontend-tests, + python-checks, + python-tests, + build, + ] if: github.event_name != 'workflow_dispatch' environment: name: pypi url: https://pypi.org/p/invokeai steps: - - name: Download distribution from build job + - name: download distribution from build job uses: actions/download-artifact@v4 with: name: dist path: dist/ - - name: Publish distribution to PyPI + - name: publish distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1