feat: automated releases via github action

- Restructure & update code check workflows
- Add release workflow to handle checks/tests, build and publish to PyPI
- Add docs/RELEASE.md explaining the workflow & process
- `create_installer.sh`: Update to work with the release workflow
- `create_installer.sh` & `tag_release.sh`: Fix the ANSI escape codes for macOS
- `tag_release.sh`: Add check for python binary name
- `tag_release.sh`: Print `git remote -v` output
- `tag_release.sh`: Fix error when deleting nonexistant tags
This commit is contained in:
psychedelicious 2023-12-24 01:16:05 +11:00 committed by Kent Keirsey
parent 3a2afe1d15
commit a0313ba634
17 changed files with 566 additions and 126 deletions

View File

@ -0,0 +1,33 @@
name: Install frontend dependencies
description: Installs frontend dependencies with pnpm, with caching
runs:
using: 'composite'
steps:
- name: Setup Node 18
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Setup pnpm
uses: pnpm/action-setup@v2
with:
version: 8
run_install: false
- name: Get pnpm store directory
shell: bash
run: |
echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
- uses: actions/cache@v3
name: Setup pnpm cache
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-store-
- name: Install frontend dependencies
run: pnpm install --prefer-frozen-lockfile
shell: bash
working-directory: invokeai/frontend/web

View File

@ -0,0 +1,11 @@
name: Install python dependencies
description: Install python dependencies with pip, with caching
runs:
using: 'composite'
steps:
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
cache-dependency-path: pyproject.toml

View File

@ -11,7 +11,7 @@ on:
- 'docker/docker-entrypoint.sh'
- 'workflows/build-container.yml'
tags:
- 'v*'
- 'v*.*.*'
workflow_dispatch:
permissions:

38
.github/workflows/check-frontend.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: 'Check: frontend'
on:
workflow_dispatch:
workflow_call:
defaults:
run:
working-directory: invokeai/frontend/web
jobs:
check-frontend:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up frontend
uses: ./.github/actions/install-frontend-deps
- name: Run tsc check
run: 'pnpm run lint:tsc'
shell: bash
- name: Run dpdm check
run: 'pnpm run lint:dpdm'
shell: bash
- name: Run eslint check
run: 'pnpm run lint:eslint'
shell: bash
- name: Run prettier check
run: 'pnpm run lint:prettier'
shell: bash
- name: Run knip check
run: 'pnpm run lint:knip'
shell: bash

View File

@ -1,15 +1,8 @@
name: Test invoke.py pip
name: 'Check: pytest'
on:
push:
branches:
- 'main'
pull_request:
types:
- 'ready_for_review'
- 'opened'
- 'synchronize'
merge_group:
workflow_dispatch:
workflow_call:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@ -17,11 +10,9 @@ concurrency:
jobs:
matrix:
if: github.event.pull_request.draft == false
strategy:
matrix:
python-version:
# - '3.9'
- '3.10'
pytorch:
- linux-cuda-11_7
@ -52,27 +43,12 @@ jobs:
env:
PIP_USE_PEP517: '1'
steps:
- name: Checkout sources
id: checkout-sources
uses: actions/checkout@v3
- name: Check for changed python files
id: changed-files
uses: tj-actions/changed-files@v41
with:
files_yaml: |
python:
- 'pyproject.toml'
- 'invokeai/**'
- '!invokeai/frontend/web/**'
- 'tests/**'
- uses: actions/checkout@v4
- name: set test prompt to main branch validation
if: steps.changed-files.outputs.python_any_changed == 'true'
run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
- name: setup python
if: steps.changed-files.outputs.python_any_changed == 'true'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
@ -80,7 +56,6 @@ jobs:
cache-dependency-path: pyproject.toml
- name: install invokeai
if: steps.changed-files.outputs.python_any_changed == 'true'
env:
PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }}
run: >
@ -88,7 +63,6 @@ jobs:
--editable=".[test]"
- name: run pytest
if: steps.changed-files.outputs.python_any_changed == 'true'
id: run-pytest
run: pytest

26
.github/workflows/check-python.yml vendored Normal file
View File

@ -0,0 +1,26 @@
name: 'Check: python'
on:
workflow_dispatch:
workflow_call:
jobs:
check-backend:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install python dependencies
uses: ./.github/actions/install-python-deps
- name: Install ruff
run: pip install ruff
shell: bash
- name: Ruff check
run: ruff check --output-format=github .
shell: bash
- name: Ruff format
run: ruff format --check .
shell: bash

View File

@ -1,45 +0,0 @@
name: Lint frontend
on:
pull_request:
types:
- 'ready_for_review'
- 'opened'
- 'synchronize'
push:
branches:
- 'main'
merge_group:
workflow_dispatch:
defaults:
run:
working-directory: invokeai/frontend/web
jobs:
lint-frontend:
if: github.event.pull_request.draft == false
runs-on: ubuntu-22.04
steps:
- name: Setup Node 18
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Checkout
uses: actions/checkout@v4
- name: Setup pnpm
uses: pnpm/action-setup@v2
with:
version: '8.12.1'
- name: Install dependencies
run: 'pnpm install --prefer-frozen-lockfile'
- name: Typescript
run: 'pnpm run lint:tsc'
- name: Madge
run: 'pnpm run lint:dpdm'
- name: ESLint
run: 'pnpm run lint:eslint'
- name: Prettier
run: 'pnpm run lint:prettier'
- name: Knip
run: 'pnpm run lint:knip'

View File

@ -0,0 +1,34 @@
name: 'On change: run check-frontend'
on:
push:
branches:
- 'main'
pull_request:
types:
- 'ready_for_review'
- 'opened'
- 'synchronize'
merge_group:
jobs:
check-changed-frontend-files:
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
outputs:
frontend_any_changed: ${{ steps.changed-files.outputs.frontend_any_changed }}
steps:
- uses: actions/checkout@v4
- name: Check for changed frontend files
id: changed-files
uses: tj-actions/changed-files@v40
with:
files_yaml: |
frontend:
- 'invokeai/frontend/web/**'
run-check-frontend:
needs: check-changed-frontend-files
if: ${{ needs.check-changed-frontend-files.outputs.frontend_any_changed == 'true' }}
uses: ./.github/workflows/check-frontend.yml

View File

@ -0,0 +1,37 @@
name: 'On change: run check-python'
on:
push:
branches:
- 'main'
pull_request:
types:
- 'ready_for_review'
- 'opened'
- 'synchronize'
merge_group:
jobs:
check-changed-python-files:
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
outputs:
python_any_changed: ${{ steps.changed-files.outputs.python_any_changed }}
steps:
- uses: actions/checkout@v4
- name: Check for changed python files
id: changed-files
uses: tj-actions/changed-files@v40
with:
files_yaml: |
python:
- 'pyproject.toml'
- 'invokeai/**'
- '!invokeai/frontend/web/**'
- 'tests/**'
run-check-python:
needs: check-changed-python-files
if: ${{ needs.check-changed-python-files.outputs.python_any_changed == 'true' }}
uses: ./.github/workflows/check-python.yml

37
.github/workflows/on-change-pytest.yml vendored Normal file
View File

@ -0,0 +1,37 @@
name: 'On change: run pytest'
on:
push:
branches:
- 'main'
pull_request:
types:
- 'ready_for_review'
- 'opened'
- 'synchronize'
merge_group:
jobs:
check-changed-python-files:
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
outputs:
python_any_changed: ${{ steps.changed-files.outputs.python_any_changed }}
steps:
- uses: actions/checkout@v4
- name: Check for changed python files
id: changed-files
uses: tj-actions/changed-files@v40
with:
files_yaml: |
python:
- 'pyproject.toml'
- 'invokeai/**'
- '!invokeai/frontend/web/**'
- 'tests/**'
run-pytest:
needs: check-changed-python-files
if: ${{ needs.check-changed-python-files.outputs.python_any_changed == 'true' }}
uses: ./.github/workflows/check-pytest.yml

103
.github/workflows/release.yml vendored Normal file
View File

@ -0,0 +1,103 @@
name: Release
on:
push:
tags:
- 'v*.*.*'
workflow_dispatch:
inputs:
skip_code_checks:
description: 'Skip code checks'
required: true
default: true
type: boolean
jobs:
check-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: samuelcolvin/check-python-version@v4
id: check-python-version
with:
version_file_path: invokeai/version/invokeai_version.py
check-frontend:
if: github.event.inputs.skip_code_checks != 'true'
uses: ./.github/workflows/check-frontend.yml
check-python:
if: github.event.inputs.skip_code_checks != 'true'
uses: ./.github/workflows/check-python.yml
check-pytest:
if: github.event.inputs.skip_code_checks != 'true'
uses: ./.github/workflows/check-pytest.yml
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install python dependencies
uses: ./.github/actions/install-python-deps
- name: Install pypa/build
run: pip install --upgrade build
- name: Setup frontend
uses: ./.github/actions/install-frontend-deps
- name: Run create_installer.sh
id: create_installer
run: ./create_installer.sh --skip_frontend_checks
working-directory: installer
- name: Upload python distribution artifact
uses: actions/upload-artifact@v4
with:
name: dist
path: ${{ steps.create_installer.outputs.DIST_PATH }}
- name: Upload installer artifact
uses: actions/upload-artifact@v4
with:
name: ${{ steps.create_installer.outputs.INSTALLER_FILENAME }}
path: ${{ steps.create_installer.outputs.INSTALLER_PATH }}
publish-testpypi:
runs-on: ubuntu-latest
needs: [check-version, check-frontend, check-python, check-pytest, build]
if: github.event_name != 'workflow_dispatch'
environment:
name: testpypi
url: https://test.pypi.org/p/invokeai
steps:
- name: Download distribution from build job
uses: actions/download-artifact@v4
with:
name: dist
path: dist/
- name: Publish distribution to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
publish-pypi:
runs-on: ubuntu-latest
needs: [check-version, check-frontend, check-python, check-pytest, build]
if: github.event_name != 'workflow_dispatch'
environment:
name: pypi
url: https://pypi.org/p/invokeai
steps:
- name: Download distribution from build job
uses: actions/download-artifact@v4
with:
name: dist
path: dist/
- name: Publish distribution to PyPI
uses: pypa/gh-action-pypi-publish@release/v1

View File

@ -1,24 +0,0 @@
name: style checks
on:
pull_request:
push:
branches: main
jobs:
ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install dependencies with pip
run: |
pip install ruff
- run: ruff check --output-format=github .
- run: ruff format --check .

View File

@ -7,7 +7,7 @@ embeddedLanguageFormatting: auto
overrides:
- files: '*.md'
options:
proseWrap: always
proseWrap: preserve
printWidth: 80
parser: markdown
cursorOffset: -1

155
docs/RELEASE.md Normal file
View File

@ -0,0 +1,155 @@
# Release Workflow
The app is published in twice, in different build formats.
- A [PyPI] distribution. This includes both a source distribution and built distribution (a wheel). Users install with `pip install invokeai`. The updater uses this build.
- An installer on the [InvokeAI Releases Page]. This is a zip file with install scripts and a wheel. This is only used for new installs.
## General Prep
Make a developer call-out for PRs to merge. Merge and test things out.
While the release workflow does not include end-to-end tests, it does pause before publishing so you can download and test the final build.
## Workflow Overview
The `release.yml` workflow runs a number of jobs to handle code checks, tests, build and publish on PyPI.
It is triggered on **tag push**, when the tag matches `v*.*.*`. It doesn't matter if you've prepped a release branch like `release/v3.5.0` or are releasing from `main` - it works the same.
!!! tip
Because commits are reference-counted, it is safe to create a release branch, tag it, let the workflow run, then delete the branch.
So long as the tag exists, that commit will exist.
### Triggering the Workflow
Run `make tag-release` to tag the current commit and kick off the workflow.
This script actually makes two tags - one for the specific version, and a `vX-latest` tag that changes with each release.
Because the release workflow only triggers on the pattern `v*.*.*`, the workflow will only run once when running this script.
The release may also be run [manually].
### Workflow Jobs and Process
The workflow consists of a number of concurrently-run jobs, and two final publish jobs.
The publish jobs run if the 5 concurrent jobs all succeed and if/when the publish jobs are approved.
#### `check-version` Job
This job checks that the git ref matches the app version. It matches the ref against the `__version__` variable in `invokeai/version/invokeai_version.py`.
When the workflow is triggered by tag push, the ref is the tag. If the workflow is run manually, the ref is the target selected from the **Use workflow from** dropdown.
!!! tip
Any valid [version specifier] works, so long as the tag matches the version. The release workflow works exactly the same for `RC`, `post`, `dev`, etc.
#### Check and Test Jobs
This is our test suite.
- **`check-pytest`**: runs `pytest` on matrix of platforms
- **`check-python`**: runs `ruff` (format and lint)
- **`check-frontend`**: runs `prettier` (format), `eslint` (lint), `madge` (circular refs) and `tsc` (static type check)
!!! info Future Enhancement
We should add `mypy` or `pyright` to the **`check-python`** job.
!!! info Future Enhancement
We should add an end-to-end test job that generates an image.
#### `build` Job
This sets up both python and frontend dependencies and builds the python package. Internally, this runs `installer/create_installer.sh` and uploads two artifacts:
- **`dist`**: the python distribution, to be published on PyPI
- **`InvokeAI-installer-${VERSION}.zip`**: the installer to be included in the GitHub release
#### Sanity Check & Smoke Test
At this point, the release workflow pauses (the remaining jobs all require approval).
A maintainer should go to the **Summary** tab of the workflow, download the installer and test it. Ensure the app loads and generates.
!!! info
The exact same wheel file is bundled in the installer and in the `dist` artifact, which is uploaded to PyPI. You should end up with the same exact installation of the `invokeai` package from any of these methods.
#### PyPI Publish Jobs
The publish jobs will skip if any of the previous jobs skip or fail.
They use [GitHub environments], which are configured as [trusted publishers] on PyPI.
Both jobs require a maintainer to approve them from the workflow's **Summary** tab.
- Click the **Review deployments** button
- Select the environment (either `testpypi` or `pypi`)
- Click **Approve and deploy**
!!! warning
**If the version already exists on PyPI, the publish jobs will fail.** PyPI only allows a particular version to be published once - you cannot change it. If version published on PyPI has a problem, you'll need to "fail forward" by bumping the app version and publishing a followup release.
#### `publish-testpypi` Job
Publishes the distribution on the [Test PyPI] index, using the `testpypi` GitHub environment.
This job is not required for the production PyPI publish, but included just in case you want to test the PyPI release.
If approved and successful, you could try out the test release like this:
```sh
# Create a new virtual environment
python -m venv ~/.test-invokeai-dist --prompt test-invokeai-dist
# Install the distribution from Test PyPI
pip install --index-url https://test.pypi.org/simple/ invokeai
# Run and test the app
invokeai-web
# Cleanup
deactivate
rm -rf ~/.test-invokeai-dist
```
#### `publish-pypi` Job
Publishes the distribution on the production PyPI index, using the `pypi` GitHub environment.
## Publish the GitHub RC with installer
1. [Draft a new release] on GitHub, choosing the tag that triggered the release.
2. Write the release notes, describing important changes. The **Generate release notes** button automatically inserts the changelog and new contributors, and you can copy/paste the intro from previous releases.
3. Upload the zip file created in [Build the installer] into the Assets section of the release notes. You can also upload the zip into the body of the release notes, since it can be hard for users to find the Assets section.
4. Check the **Set as a pre-release** and **Create a discussion for this release** checkboxes at the bottom of the release page.
5. Publish the pre-release.
6. Announce the pre-release in Discord.
!!! info Future Enhancement
Workflows can create a GitHub release from a template and upload release assets. One popular action to handle this is [ncipollo/release-action]. A future enhancement to the release process could set this up.
## Manually Running the Release Workflow
The release workflow can be run manually. This is useful to get an installer build and test it out without needing to push a tag.
When run this way, you'll see **Skip code checks** checkbox. This allows the workflow to run without the time-consuming 3 code quality check jobs.
The publish jobs will skip if the workflow was run manually.
[InvokeAI Releases Page]: https://github.com/invoke-ai/InvokeAI/releases
[PyPI]: https://pypi.org/
[Draft a new release]: https://github.com/invoke-ai/InvokeAI/releases/new
[Test PyPI]: https://test.pypi.org/
[version specifier]: https://packaging.python.org/en/latest/specifications/version-specifiers/
[ncipollo/release-action]: https://github.com/ncipollo/release-action
[GitHub environments]: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment
[trusted publishers]: https://docs.pypi.org/trusted-publishers/
[samuelcolvin/check-python-version]: https://github.com/samuelcolvin/check-python-version
[manually]: #manually-running-the-release-workflow

View File

@ -2,12 +2,12 @@
set -e
BCYAN="\e[1;36m"
BYELLOW="\e[1;33m"
BGREEN="\e[1;32m"
BRED="\e[1;31m"
RED="\e[31m"
RESET="\e[0m"
BCYAN="\033[1;36m"
BYELLOW="\033[1;33m"
BGREEN="\033[1;32m"
BRED="\033[1;31m"
RED="\033[31m"
RESET="\033[0m"
function is_bin_in_path {
builtin type -P "$1" &>/dev/null
@ -44,13 +44,47 @@ VERSION=$(
cd ..
python -c "from invokeai.version import __version__ as version; print(version)"
)
PATCH=""
VERSION="v${VERSION}${PATCH}"
VERSION="v${VERSION}"
echo -e "${BGREEN}HEAD${RESET}:"
git_show HEAD
echo
# ---------------------- FRONTEND ----------------------
pushd ../invokeai/frontend/web >/dev/null
echo
echo "Installing frontend dependencies..."
echo
pnpm i --frozen-lockfile
echo
echo "Building frontend..."
if [[ -v CI ]]; then
# In CI, we have already done the frontend checks and can just build
pnpm vite build
else
# This runs all the frontend checks and builds
pnpm build
fi
echo
popd
# ---------------------- BACKEND ----------------------
echo
echo "Building wheel..."
echo
# install the 'build' package in the user site packages, if needed
# could be improved by using a temporary venv, but it's tiny and harmless
if [[ $(python -c 'from importlib.util import find_spec; print(find_spec("build") is None)') == "True" ]]; then
pip install --user build
fi
rm -rf ../build
python -m build --outdir dist/ ../.
# ----------------------
echo
@ -78,10 +112,21 @@ chmod a+x InvokeAI-Installer/install.sh
cp install.bat.in InvokeAI-Installer/install.bat
cp WinLongPathsEnabled.reg InvokeAI-Installer/
# Zip everything up
zip -r InvokeAI-installer-$VERSION.zip InvokeAI-Installer
FILENAME=InvokeAI-installer-$VERSION.zip
# clean up
# Zip everything up
zip -r $FILENAME InvokeAI-Installer
if [[ ! -v CI ]]; then
# clean up, but only if we are not in a github action
rm -rf InvokeAI-Installer tmp dist ../invokeai/frontend/web/dist/
fi
if [[ -v CI ]]; then
# Set the output variable for github action
echo "INSTALLER_FILENAME=$FILENAME" >>$GITHUB_OUTPUT
echo "INSTALLER_PATH=installer/$FILENAME" >>$GITHUB_OUTPUT
echo "DIST_PATH=installer/dist/" >>$GITHUB_OUTPUT
fi
exit 0

View File

@ -2,12 +2,16 @@
set -e
BCYAN="\e[1;36m"
BYELLOW="\e[1;33m"
BGREEN="\e[1;32m"
BRED="\e[1;31m"
RED="\e[31m"
RESET="\e[0m"
BCYAN="\033[1;36m"
BYELLOW="\033[1;33m"
BGREEN="\033[1;32m"
BRED="\033[1;31m"
RED="\033[31m"
RESET="\033[0m"
function is_bin_in_path {
builtin type -P "$1" &>/dev/null
}
function does_tag_exist {
git rev-parse --quiet --verify "refs/tags/$1" >/dev/null
@ -21,6 +25,14 @@ function git_show {
git show -s --format='%h %s' $1
}
# Some machines only have `python3` in PATH, others have `python` - make an alias.
# We can use a function to approximate an alias within a non-interactive shell.
if ! is_bin_in_path python && is_bin_in_path python3; then
function python {
python3 "$@"
}
fi
VERSION=$(
cd ..
python -c "from invokeai.version import __version__ as version; print(version)"
@ -45,27 +57,31 @@ echo -e "${BGREEN}HEAD${RESET}:"
git_show
echo
echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} and ${BCYAN}${LATEST_TAG}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on remote${RESET}? "
echo -e "${BGREEN}git remote -v${RESET}:"
git remote -v
echo
echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} and ${BCYAN}${LATEST_TAG}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on origin remote${RESET}? "
read -e -p 'y/n [n]: ' input
RESPONSE=${input:='n'}
if [ "$RESPONSE" == 'y' ]; then
echo
echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on remote..."
git push --delete origin $VERSION
echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on origin remote..."
git push origin :refs/tags/$VERSION
echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} locally..."
echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} on locally..."
if ! git tag -fa $VERSION; then
echo "Existing/invalid tag"
exit -1
fi
echo -e "Deleting ${BCYAN}${LATEST_TAG}${RESET} tag on remote..."
git push --delete origin $LATEST_TAG
echo -e "Deleting ${BCYAN}${LATEST_TAG}${RESET} tag on origin remote..."
git push origin :refs/tags/$LATEST_TAG
echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${LATEST_TAG}${RESET} locally..."
git tag -fa $LATEST_TAG
echo -e "Pushing updated tags to remote..."
echo -e "Pushing updated tags to origin remote..."
git push origin --tags
fi
exit 0