From b9fd263899253329a6e1cdae9a155120a1458f6a Mon Sep 17 00:00:00 2001 From: Oliver Date: Sun, 29 May 2022 09:40:37 +1000 Subject: [PATCH] Docker improvements (#3042) * Simplified dockerfile - Changed from alpine to python:slim - Removed some database libs (because we *connect* to a db, not host it) * - Add gettext as required package - Only create inventree user as part of production build (leave admin access for dev build) * Tweaks for tasks.py * Fix user permissions (drop to inventree user) * Drop to the 'inventree' user level as part of init.sh - As we have mounted volumes at 'run time' we need to ensure that the inventree user has correct permissions! - Ref: https://stackoverflow.com/questions/39397548/how-to-give-non-root-user-in-docker-container-access-to-a-volume-mounted-on-the * Adjust user setup - Only drop to non-root user as part of "production" build - Mounted external volumes make it tricky when in the dev build - Might want to revisit this later on * More dockerfile changes - reduce required system packages - * Add new docker github workflow * Print some more debug * GITHUB_BASE_REF * Add gnupg to base requirements * Improve debug output during testing * Refactoring updates for label printing API - Update weasyprint version to 55.0 - Generate labels as pdf files - Provide filename to label printing plugin - Additional unit testing - Improve extraction of some hidden debug data during TESTING - Fix a spelling mistake (notifaction -> notification) * Working on github action * More testing * Add requirement for pdf2image * Fix label printing plugin and update unit testing * Add required packages for CI * Move docker files to the top level directory - This allows us to build the production image directly from soure - Don't need to re-download the source code from github - Note: The docker install guide will need to be updated! * Fix for docker ci file * Print GIT SHA * Bake git information into the production image * Add some exta docstrings to dockerfile * Simplify version check script * Extract git commit info * Extract docker tag from check_version.py * Newline * More work on the docker workflow * Dockerfile fixes - Directory / path issues * Dockerfile fixes - Directory / path issues * Ignore certain steps on a pull request * Add poppler-utils to CI * Consolidate version check into existing CI file * Don't run docker workflow on pull request * Pass docker image tag through to the build Also check .j2k files * Add supervisord.conf example file back in * Remove --no-cache-dir option from pip install --- docker/.env => .env | 1 + .github/workflows/docker.yaml | 69 ++++++++++ .github/workflows/docker_latest.yaml | 51 ------- .github/workflows/docker_stable.yaml | 42 ------ .github/workflows/docker_tag.yaml | 38 ------ .github/workflows/qc_checks.yaml | 11 +- .github/workflows/version.yml | 21 --- .gitignore | 5 +- docker/Dockerfile => Dockerfile | 125 ++++++++--------- InvenTree/InvenTree/api_tester.py | 5 + InvenTree/InvenTree/exceptions.py | 6 +- InvenTree/InvenTree/helpers.py | 2 +- InvenTree/InvenTree/tests.py | 24 ++++ InvenTree/InvenTree/version.py | 13 ++ InvenTree/common/notifications.py | 2 +- InvenTree/label/api.py | 17 +-- InvenTree/part/tasks.py | 2 +- InvenTree/part/test_api.py | 2 +- InvenTree/plugin/base/label/label.py | 51 ++++++- InvenTree/plugin/base/label/mixins.py | 15 ++- .../plugin/base/label/test_label_mixin.py | 20 ++- .../samples/integration/label_sample.py | 21 ++- ci/check_version_number.py | 127 ++++++++++-------- .../docker-compose.yml => docker-compose.yml | 2 +- docker/init.sh | 2 +- docker/production/.env | 6 + requirements.txt | 6 +- tasks.py | 4 +- 28 files changed, 376 insertions(+), 314 deletions(-) rename docker/.env => .env (89%) create mode 100644 .github/workflows/docker.yaml delete mode 100644 .github/workflows/docker_latest.yaml delete mode 100644 .github/workflows/docker_stable.yaml delete mode 100644 .github/workflows/docker_tag.yaml delete mode 100644 .github/workflows/version.yml rename docker/Dockerfile => Dockerfile (53%) rename docker/docker-compose.yml => docker-compose.yml (98%) diff --git a/docker/.env b/.env similarity index 89% rename from docker/.env rename to .env index 54e37ea7a0..586b0daab3 100644 --- a/docker/.env +++ b/.env @@ -1,4 +1,5 @@ # InvenTree environment variables for a development setup +# These variables will be used by the docker-compose.yml file # Set DEBUG to True for a development setup INVENTREE_DEBUG=True diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000000..b9f895983d --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,69 @@ +# Build, test and push InvenTree docker image +# This workflow runs under any of the following conditions: +# +# - Push to the master branch +# - Push to the stable branch +# - Publish release +# +# The following actions are performed: +# +# - Check that the version number matches the current branch or tag +# - Build the InvenTree docker image +# - Run suite of unit tests against the build image +# - Push the compiled, tested image to dockerhub + +name: Docker + +on: + release: + types: [published] + + push: + branches: + - 'master' + - 'stable' + +jobs: + + # Build the docker image + build: + runs-on: ubuntu-latest + + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Version Check + run: | + python3 ci/check_version_number.py + echo "git_commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "git_commit_date=$(git show -s --format=%ci)" >> $GITHUB_ENV + - name: Run Unit Tests + run: | + docker-compose build + docker-compose run inventree-dev-server invoke update + docker-compose up -d + docker-compose run inventree-dev-server invoke wait + docker-compose run inventree-dev-server invoke test + docker-compose down + - name: Set up QEMU + if: github.event_name != 'pull_request' + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + if: github.event_name != 'pull_request' + uses: docker/setup-buildx-action@v1 + - name: Login to Dockerhub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and Push + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: false + target: production + tags: inventree/inventree:${{ env.docker_tag }} + build-args: commit_hash=${{ env.git_commit_hash }},commit_date=${{ env.git_commit_date }},commit_tag=${{ env.docker_tag }} diff --git a/.github/workflows/docker_latest.yaml b/.github/workflows/docker_latest.yaml deleted file mode 100644 index 74b5eb966c..0000000000 --- a/.github/workflows/docker_latest.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Build and push latest docker image on push to master branch - -name: Docker Build - -on: - push: - branches: - - 'master' - -jobs: - - docker: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Check version number - run: | - python3 ci/check_version_number.py --dev - - name: Build Docker Image - run: | - cd docker - docker-compose build - docker-compose run inventree-dev-server invoke update - - name: Run unit tests - run: | - cd docker - docker-compose up -d - docker-compose run inventree-dev-server invoke wait - docker-compose run inventree-dev-server invoke test - docker-compose down - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Dockerhub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./docker - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: true - target: production - tags: inventree/inventree:latest - - name: Image Digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker_stable.yaml b/.github/workflows/docker_stable.yaml deleted file mode 100644 index e892b24d13..0000000000 --- a/.github/workflows/docker_stable.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Build and push docker image on push to 'stable' branch -# Docker build will be uploaded to dockerhub with the 'inventree:stable' tag - -name: Docker Build - -on: - push: - branches: - - 'stable' - -jobs: - - docker: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Check version number - run: | - python3 ci/check_version_number.py --release - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Dockerhub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./docker - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: true - target: production - build-args: - branch=stable - tags: inventree/inventree:stable - - name: Image Digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker_tag.yaml b/.github/workflows/docker_tag.yaml deleted file mode 100644 index a9f1c646fc..0000000000 --- a/.github/workflows/docker_tag.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Publish docker images to dockerhub on a tagged release -# Docker build will be uploaded to dockerhub with the 'invetree:' tag - -name: Docker Publish - -on: - release: - types: [published] - -jobs: - publish_image: - name: Push InvenTree web server image to dockerhub - runs-on: ubuntu-latest - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Check Release tag - run: | - python3 ci/check_version_number.py --release --tag ${{ github.event.release.tag_name }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Dockerhub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push - uses: docker/build-push-action@v2 - with: - context: ./docker - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: true - target: production - build-args: - tag=${{ github.event.release.tag_name }} - tags: inventree/inventree:${{ github.event.release.tag_name }} diff --git a/.github/workflows/qc_checks.yaml b/.github/workflows/qc_checks.yaml index cf2700c3d3..9bc0484fa0 100644 --- a/.github/workflows/qc_checks.yaml +++ b/.github/workflows/qc_checks.yaml @@ -91,6 +91,9 @@ jobs: cache: 'pip' - name: Run pre-commit Checks uses: pre-commit/action@v2.0.3 + - name: Check version number + run: | + python3 ci/check_version_number.py python: name: Tests - inventree-python @@ -114,7 +117,7 @@ jobs: - name: Enviroment Setup uses: ./.github/actions/setup with: - apt-dependency: gettext + apt-dependency: gettext poppler-utils update: true - name: Download Python Code For `${{ env.wrapper_name }}` run: git clone --depth 1 https://github.com/inventree/${{ env.wrapper_name }} ./${{ env.wrapper_name }} @@ -147,7 +150,7 @@ jobs: - name: Enviroment Setup uses: ./.github/actions/setup with: - apt-dependency: gettext + apt-dependency: gettext poppler-utils update: true - name: Coverage Tests run: invoke coverage @@ -196,7 +199,7 @@ jobs: - name: Enviroment Setup uses: ./.github/actions/setup with: - apt-dependency: gettext libpq-dev + apt-dependency: gettext poppler-utils libpq-dev pip-dependency: psycopg2 django-redis>=5.0.0 update: true - name: Run Tests @@ -239,7 +242,7 @@ jobs: - name: Enviroment Setup uses: ./.github/actions/setup with: - apt-dependency: gettext libmysqlclient-dev + apt-dependency: gettext poppler-utils libmysqlclient-dev pip-dependency: mysqlclient update: true - name: Run Tests diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml deleted file mode 100644 index 73d5bd8a2c..0000000000 --- a/.github/workflows/version.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Checks version number -name: version number - -on: - pull_request: - branches-ignore: - - l10* - - -jobs: - - check_version: - name: version number - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Check version number - run: | - python3 ci/check_version_number.py --branch ${{ github.base_ref }} diff --git a/.gitignore b/.gitignore index 56d4180482..9c9a45d136 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ env/ inventree-env/ ./build/ +.cache/ develop-eggs/ dist/ bin/ @@ -26,7 +27,6 @@ var/ .installed.cfg *.egg - # Django stuff: *.log local_settings.py @@ -38,6 +38,8 @@ local_settings.py # Files used for testing dummy_image.* _tmp.csv +inventree/label.pdf +inventree/label.png # Sphinx files docs/_build @@ -63,6 +65,7 @@ secret_key.txt .idea/ *.code-workspace .vscode/ +.bash_history # Coverage reports .coverage diff --git a/docker/Dockerfile b/Dockerfile similarity index 53% rename from docker/Dockerfile rename to Dockerfile index 1b7c16db30..361ef686f0 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -1,37 +1,39 @@ -FROM alpine:3.14 as base +# The InvenTree dockerfile provides two build targets: +# +# production: +# - Required files are copied into the image +# - Runs InvenTree web server under gunicorn +# +# dev: +# - Expects source directories to be loaded as a run-time volume +# - Runs InvenTree web server under django development server +# - Monitors source files for any changes, and live-reloads server -# GitHub source -ARG repository="https://github.com/inventree/InvenTree.git" -ARG branch="master" -# Optionally specify a particular tag to checkout -ARG tag="" +FROM python:3.9-slim as base + +# Build arguments for this image +ARG commit_hash="" +ARG commit_date="" +ARG commit_tag="" ENV PYTHONUNBUFFERED 1 # Ref: https://github.com/pyca/cryptography/issues/5776 ENV CRYPTOGRAPHY_DONT_BUILD_RUST 1 -# InvenTree key settings - -# The INVENTREE_HOME directory is where the InvenTree source repository will be located -ENV INVENTREE_HOME="/home/inventree" - -# GitHub settings -ENV INVENTREE_GIT_REPO="${repository}" -ENV INVENTREE_GIT_BRANCH="${branch}" -ENV INVENTREE_GIT_TAG="${tag}" - ENV INVENTREE_LOG_LEVEL="INFO" ENV INVENTREE_DOCKER="true" # InvenTree paths +ENV INVENTREE_HOME="/home/inventree" ENV INVENTREE_MNG_DIR="${INVENTREE_HOME}/InvenTree" ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data" ENV INVENTREE_STATIC_ROOT="${INVENTREE_DATA_DIR}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media" ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DATA_DIR}/plugins" +# InvenTree configuration files ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt" ENV INVENTREE_PLUGIN_FILE="${INVENTREE_DATA_DIR}/plugins.txt" @@ -49,82 +51,83 @@ LABEL org.label-schema.schema-version="1.0" \ org.label-schema.vendor="inventree" \ org.label-schema.name="inventree/inventree" \ org.label-schema.url="https://hub.docker.com/r/inventree/inventree" \ - org.label-schema.vcs-url=${INVENTREE_GIT_REPO} \ - org.label-schema.vcs-branch=${INVENTREE_GIT_BRANCH} \ - org.label-schema.vcs-ref=${INVENTREE_GIT_TAG} + org.label-schema.vcs-url="https://github.com/inventree/InvenTree.git" \ + org.label-schema.vcs-ref=${commit_tag} -# Create user account -RUN addgroup -S inventreegroup && adduser -S inventree -G inventreegroup - -RUN apk -U upgrade +# RUN apt-get upgrade && apt-get update +RUN apt-get update # Install required system packages -RUN apk add --no-cache git make bash \ - gcc libgcc g++ libstdc++ \ - gnupg \ - libjpeg-turbo libjpeg-turbo-dev jpeg jpeg-dev libwebp-dev \ - libffi libffi-dev \ - zlib zlib-dev \ - # Special deps for WeasyPrint (these will be deprecated once WeasyPrint drops cairo requirement) - cairo cairo-dev pango pango-dev gdk-pixbuf \ - # Fonts - fontconfig ttf-droid ttf-liberation ttf-dejavu ttf-opensans font-croscore font-noto \ - # Core python - python3 python3-dev py3-pip \ +RUN apt-get install -y --no-install-recommends \ + git gcc g++ gettext gnupg \ + # Weasyprint requirements : https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#debian-11 + poppler-utils libpango-1.0-0 libpangoft2-1.0-0 \ + # Image format support + libjpeg-dev webp \ # SQLite support - sqlite \ + sqlite3 \ # PostgreSQL support - postgresql postgresql-contrib postgresql-dev libpq \ - # MySQL/MariaDB support - mariadb-connector-c mariadb-dev mariadb-client \ - # Required for python cryptography support - openssl-dev musl-dev libffi-dev rust cargo + libpq-dev \ + # MySQL / MariaDB support + default-libmysqlclient-dev mariadb-client && \ + apt-get autoclean && apt-get autoremove # Update pip RUN pip install --upgrade pip # Install required base-level python packages -COPY requirements.txt requirements.txt -RUN pip install --no-cache-dir -U -r requirements.txt +COPY ./docker/requirements.txt base_requirements.txt +RUN pip install --disable-pip-version-check -U -r base_requirements.txt + +# InvenTree production image: +# - Copies required files from local directory +# - Installs required python packages from requirements.txt +# - Starts a gunicorn webserver -# Production code (pulled from tagged github release) FROM base as production -# Clone source code -RUN echo "Downloading InvenTree from ${INVENTREE_GIT_REPO}" +ENV INVENTREE_DEBUG=False -RUN git clone --branch ${INVENTREE_GIT_BRANCH} --depth 1 ${INVENTREE_GIT_REPO} ${INVENTREE_HOME} +# As .git directory is not available in production image, we pass the commit information via ENV +ENV INVENTREE_COMMIT_HASH="${commit_hash}" +ENV INVENTREE_COMMIT_DATE="${commit_date}" -# Ref: https://github.blog/2022-04-12-git-security-vulnerability-announced/ -RUN git config --global --add safe.directory ${INVENTREE_HOME} +# Copy source code +COPY InvenTree ${INVENTREE_HOME}/InvenTree -# Checkout against a particular git tag -RUN if [ -n "${INVENTREE_GIT_TAG}" ] ; then cd ${INVENTREE_HOME} && git fetch --all --tags && git checkout tags/${INVENTREE_GIT_TAG} -b v${INVENTREE_GIT_TAG}-branch ; fi - -RUN chown -R inventree:inventreegroup ${INVENTREE_HOME}/* - -# Drop to the inventree user -USER inventree - -# Install InvenTree packages -RUN pip3 install --user --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt +# Copy other key files +COPY requirements.txt ${INVENTREE_HOME}/requirements.txt +COPY tasks.py ${INVENTREE_HOME}/tasks.py +COPY docker/gunicorn.conf.py ${INVENTREE_HOME}/gunicorn.conf.py +COPY docker/init.sh ${INVENTREE_MNG_DIR}/init.sh # Need to be running from within this directory WORKDIR ${INVENTREE_MNG_DIR} +# Drop to the inventree user for the production image +RUN adduser inventree +RUN chown -R inventree:inventree ${INVENTREE_HOME} + +USER inventree + +# Install InvenTree packages +RUN pip3 install --user --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt + # Server init entrypoint -ENTRYPOINT ["/bin/bash", "../docker/init.sh"] +ENTRYPOINT ["/bin/bash", "./init.sh"] # Launch the production server # TODO: Work out why environment variables cannot be interpolated in this command # TODO: e.g. -b ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} fails here -CMD gunicorn -c ./docker/gunicorn.conf.py InvenTree.wsgi -b 0.0.0.0:8000 --chdir ./InvenTree +CMD gunicorn -c ./gunicorn.conf.py InvenTree.wsgi -b 0.0.0.0:8000 --chdir ./InvenTree FROM base as dev # The development image requires the source code to be mounted to /home/inventree/ # So from here, we don't actually "do" anything, apart from some file management +ENV INVENTREE_DEBUG=True + ENV INVENTREE_DEV_DIR="${INVENTREE_HOME}/dev" # Location for python virtual environment diff --git a/InvenTree/InvenTree/api_tester.py b/InvenTree/InvenTree/api_tester.py index 935252de5b..5385f8f01b 100644 --- a/InvenTree/InvenTree/api_tester.py +++ b/InvenTree/InvenTree/api_tester.py @@ -117,6 +117,11 @@ class InvenTreeAPITestCase(UserMixin, APITestCase): response = self.client.get(url, data, format='json') if expected_code is not None: + + if response.status_code != expected_code: + print(f"Unexpected response at '{url}':") + print(response.data) + self.assertEqual(response.status_code, expected_code) return response diff --git a/InvenTree/InvenTree/exceptions.py b/InvenTree/InvenTree/exceptions.py index 55017affc0..a4737bac4d 100644 --- a/InvenTree/InvenTree/exceptions.py +++ b/InvenTree/InvenTree/exceptions.py @@ -40,7 +40,11 @@ def exception_handler(exc, context): if response is None: # DRF handler did not provide a default response for this exception - if settings.DEBUG: + if settings.TESTING: + # If in TESTING mode, re-throw the exception for traceback + raise exc + elif settings.DEBUG: + # If in DEBUG mode, provide error information in the response error_detail = str(exc) else: error_detail = _("Error details can be found in the admin panel") diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index c541ce4ef5..00ac33ae68 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -129,7 +129,7 @@ def TestIfImageURL(url): Simply tests the extension against a set of allowed values """ return os.path.splitext(os.path.basename(url))[-1].lower() in [ - '.jpg', '.jpeg', + '.jpg', '.jpeg', '.j2k', '.png', '.bmp', '.tif', '.tiff', '.webp', '.gif', diff --git a/InvenTree/InvenTree/tests.py b/InvenTree/InvenTree/tests.py index 61130ad1b4..3de293ca66 100644 --- a/InvenTree/InvenTree/tests.py +++ b/InvenTree/InvenTree/tests.py @@ -380,6 +380,30 @@ class TestVersionNumber(TestCase): self.assertTrue(v_d > v_c) self.assertTrue(v_d > v_a) + def test_commit_info(self): + """Test that the git commit information is extracted successfully""" + + envs = { + 'INVENTREE_COMMIT_HASH': 'abcdef', + 'INVENTREE_COMMIT_DATE': '2022-12-31' + } + + # Check that the environment variables take priority + + with mock.patch.dict(os.environ, envs): + self.assertEqual(version.inventreeCommitHash(), 'abcdef') + self.assertEqual(version.inventreeCommitDate(), '2022-12-31') + + import subprocess + + # Check that the current .git values work too + + hash = str(subprocess.check_output('git rev-parse --short HEAD'.split()), 'utf-8').strip() + self.assertEqual(hash, version.inventreeCommitHash()) + + d = str(subprocess.check_output('git show -s --format=%ci'.split()), 'utf-8').strip().split(' ')[0] + self.assertEqual(d, version.inventreeCommitDate()) + class CurrencyTests(TestCase): """ diff --git a/InvenTree/InvenTree/version.py b/InvenTree/InvenTree/version.py index f1190ec7ba..fe970ee5d4 100644 --- a/InvenTree/InvenTree/version.py +++ b/InvenTree/InvenTree/version.py @@ -3,6 +3,7 @@ Version information for InvenTree. Provides information on the current InvenTree version """ +import os import re import subprocess @@ -99,6 +100,12 @@ def inventreeDjangoVersion(): def inventreeCommitHash(): """ Returns the git commit hash for the running codebase """ + # First look in the environment variables, i.e. if running in docker + commit_hash = os.environ.get('INVENTREE_COMMIT_HASH', '') + + if commit_hash: + return commit_hash + try: return str(subprocess.check_output('git rev-parse --short HEAD'.split()), 'utf-8').strip() except: # pragma: no cover @@ -108,6 +115,12 @@ def inventreeCommitHash(): def inventreeCommitDate(): """ Returns the git commit date for the running codebase """ + # First look in the environment variables, e.g. if running in docker + commit_date = os.environ.get('INVENTREE_COMMIT_DATE', '') + + if commit_date: + return commit_date.split(' ')[0] + try: d = str(subprocess.check_output('git show -s --format=%ci'.split()), 'utf-8').strip() return d.split(' ')[0] diff --git a/InvenTree/common/notifications.py b/InvenTree/common/notifications.py index c2e4f2d6aa..aa39ad20ef 100644 --- a/InvenTree/common/notifications.py +++ b/InvenTree/common/notifications.py @@ -203,7 +203,7 @@ class UIMessageNotification(SingleNotificationMethod): return True -def trigger_notifaction(obj, category=None, obj_ref='pk', **kwargs): +def trigger_notification(obj, category=None, obj_ref='pk', **kwargs): """ Send out a notification """ diff --git a/InvenTree/label/api.py b/InvenTree/label/api.py index cb7ea12598..7988bfb32f 100644 --- a/InvenTree/label/api.py +++ b/InvenTree/label/api.py @@ -1,12 +1,9 @@ -from io import BytesIO - from django.conf import settings from django.core.exceptions import FieldError, ValidationError from django.http import HttpResponse, JsonResponse from django.urls import include, re_path from django_filters.rest_framework import DjangoFilterBackend -from PIL import Image from rest_framework import filters, generics from rest_framework.exceptions import NotFound @@ -137,25 +134,21 @@ class LabelPrintMixin: # Label instance label_instance = self.get_object() - for output in outputs: + for idx, output in enumerate(outputs): """ For each output, we generate a temporary image file, which will then get sent to the printer """ - # Generate a png image at 300dpi - (img_data, w, h) = output.get_document().write_png(resolution=300) - - # Construct a BytesIO object, which can be read by pillow - img_bytes = BytesIO(img_data) - - image = Image.open(img_bytes) + # Generate PDF data for the label + pdf = output.get_document().write_pdf() # Offload a background task to print the provided label offload_task( plugin_label.print_label, plugin.plugin_slug(), - image, + pdf, + filename=label_names[idx], label_instance=label_instance, user=request.user, ) diff --git a/InvenTree/part/tasks.py b/InvenTree/part/tasks.py index 18188bed77..9037fb92be 100644 --- a/InvenTree/part/tasks.py +++ b/InvenTree/part/tasks.py @@ -24,7 +24,7 @@ def notify_low_stock(part: part.models.Part): }, } - common.notifications.trigger_notifaction( + common.notifications.trigger_notification( part, 'part.notify_low_stock', target_fnc=part.get_subscribers, diff --git a/InvenTree/part/test_api.py b/InvenTree/part/test_api.py index 3819a01706..02fbe5d29e 100644 --- a/InvenTree/part/test_api.py +++ b/InvenTree/part/test_api.py @@ -1098,7 +1098,7 @@ class PartDetailTests(InvenTreeAPITestCase): self.assertIn('Upload a valid image', str(response.data)) # Now try to upload a valid image file, in multiple formats - for fmt in ['jpg', 'png', 'bmp', 'webp']: + for fmt in ['jpg', 'j2k', 'png', 'bmp', 'webp']: fn = f'dummy_image.{fmt}' img = PIL.Image.new('RGB', (128, 128), color='red') diff --git a/InvenTree/plugin/base/label/label.py b/InvenTree/plugin/base/label/label.py index d85251dd81..56eaf1bc20 100644 --- a/InvenTree/plugin/base/label/label.py +++ b/InvenTree/plugin/base/label/label.py @@ -1,7 +1,14 @@ """Functions to print a label to a mixin printer""" import logging +import sys +import traceback +from django.conf import settings from django.utils.translation import gettext_lazy as _ +from django.views.debug import ExceptionReporter + +import pdf2image +from error_report.models import Error import common.notifications from plugin.registry import registry @@ -9,7 +16,7 @@ from plugin.registry import registry logger = logging.getLogger('inventree') -def print_label(plugin_slug, label_image, label_instance=None, user=None): +def print_label(plugin_slug, pdf_data, filename=None, label_instance=None, user=None): """ Print label with the provided plugin. @@ -19,10 +26,11 @@ def print_label(plugin_slug, label_image, label_instance=None, user=None): Arguments: plugin_slug: The unique slug (key) of the plugin - label_image: A PIL.Image image object to be printed + pdf_data: Binary PDF data + filename: The intended name of the printed label """ - logger.info(f"Plugin '{plugin_slug}' is printing a label") + logger.info(f"Plugin '{plugin_slug}' is printing a label '{filename}'") plugin = registry.plugins.get(plugin_slug, None) @@ -30,8 +38,22 @@ def print_label(plugin_slug, label_image, label_instance=None, user=None): logger.error(f"Could not find matching plugin for '{plugin_slug}'") return + # In addition to providing a .pdf image, we'll also provide a .png file + png_file = pdf2image.convert_from_bytes( + pdf_data, + dpi=300, + )[0] + try: - plugin.print_label(label_image, width=label_instance.width, height=label_instance.height) + plugin.print_label( + pdf_data=pdf_data, + png_file=png_file, + filename=filename, + label_instance=label_instance, + width=label_instance.width, + height=label_instance.height, + user=user + ) except Exception as e: # pragma: no cover # Plugin threw an error - notify the user who attempted to print @@ -40,13 +62,28 @@ def print_label(plugin_slug, label_image, label_instance=None, user=None): 'message': str(e), } - logger.error(f"Label printing failed: Sending notification to user '{user}'") + # Log an error message to the database + kind, info, data = sys.exc_info() + + Error.objects.create( + kind=kind.__name__, + info=info, + data='\n'.join(traceback.format_exception(kind, info, data)), + path='print_label', + html=ExceptionReporter(None, kind, info, data).get_traceback_html(), + ) + + logger.error(f"Label printing failed: Sending notification to user '{user}'") # pragma: no cover # Throw an error against the plugin instance - common.notifications.trigger_notifaction( + common.notifications.trigger_notification( plugin.plugin_config(), 'label.printing_failed', targets=[user], context=ctx, - delivery_methods=[common.notifications.UIMessageNotification] + delivery_methods=set([common.notifications.UIMessageNotification]) ) + + if settings.TESTING: + # If we are in testing mode, we want to know about this exception + raise e diff --git a/InvenTree/plugin/base/label/mixins.py b/InvenTree/plugin/base/label/mixins.py index 4e06f9e15a..aa17b1812a 100644 --- a/InvenTree/plugin/base/label/mixins.py +++ b/InvenTree/plugin/base/label/mixins.py @@ -22,17 +22,18 @@ class LabelPrintingMixin: super().__init__() self.add_mixin('labels', True, __class__) - def print_label(self, label, **kwargs): + def print_label(self, **kwargs): """ Callback to print a single label - Arguments: - label: A black-and-white pillow Image object - kwargs: - length: The length of the label (in mm) - width: The width of the label (in mm) - + pdf_data: Raw PDF data of the rendered label + png_file: An in-memory PIL image file, rendered at 300dpi + label_instance: The instance of the label model which triggered the print_label() method + width: The expected width of the label (in mm) + height: The expected height of the label (in mm) + filename: The filename of this PDF label + user: The user who printed this label """ # Unimplemented (to be implemented by the particular plugin class) diff --git a/InvenTree/plugin/base/label/test_label_mixin.py b/InvenTree/plugin/base/label/test_label_mixin.py index 29250f76a5..53294d2f24 100644 --- a/InvenTree/plugin/base/label/test_label_mixin.py +++ b/InvenTree/plugin/base/label/test_label_mixin.py @@ -1,8 +1,11 @@ """Unit tests for the label printing mixin""" +import os from django.apps import apps from django.urls import reverse +from PIL import Image + from common.models import InvenTreeSetting from InvenTree.api_tester import InvenTreeAPITestCase from label.models import PartLabel, StockItemLabel, StockLocationLabel @@ -68,7 +71,7 @@ class LabelMixinTests(InvenTreeAPITestCase): with self.assertRaises(MixinNotImplementedError): plugin = WrongPlugin() - plugin.print_label('test') + plugin.print_label(filename='test') def test_installed(self): """Test that the sample printing plugin is installed""" @@ -167,6 +170,21 @@ class LabelMixinTests(InvenTreeAPITestCase): # Print no part self.get(self.do_url(None, plugin_ref, label), expected_code=400) + # Test that the labels have been printed + # The sample labelling plugin simply prints to file + self.assertTrue(os.path.exists('label.pdf')) + + # Read the raw .pdf data - ensure it contains some sensible information + with open('label.pdf', 'rb') as f: + pdf_data = str(f.read()) + self.assertIn('WeasyPrint', pdf_data) + + # Check that the .png file has already been created + self.assertTrue(os.path.exists('label.png')) + + # And that it is a valid image file + Image.open('label.png') + def test_printing_endpoints(self): """Cover the endpoints not covered by `test_printing_process`""" plugin_ref = 'samplelabel' diff --git a/InvenTree/plugin/samples/integration/label_sample.py b/InvenTree/plugin/samples/integration/label_sample.py index 845e1b7908..c01c575012 100644 --- a/InvenTree/plugin/samples/integration/label_sample.py +++ b/InvenTree/plugin/samples/integration/label_sample.py @@ -12,7 +12,22 @@ class SampleLabelPrinter(LabelPrintingMixin, InvenTreePlugin): SLUG = "samplelabel" TITLE = "Sample Label Printer" DESCRIPTION = "A sample plugin which provides a (fake) label printer interface" - VERSION = "0.1" + VERSION = "0.2" - def print_label(self, label, **kwargs): - print("OK PRINTING") + def print_label(self, **kwargs): + + # Test that the expected kwargs are present + print(f"Printing Label: {kwargs['filename']} (User: {kwargs['user']})") + print(f"Width: {kwargs['width']} x Height: {kwargs['height']}") + + pdf_data = kwargs['pdf_data'] + png_file = kwargs['png_file'] + + filename = kwargs['filename'] + + # Dump the PDF to a local file + with open(filename, 'wb') as pdf_out: + pdf_out.write(pdf_data) + + # Save the PNG to disk + png_file.save(filename.replace('.pdf', '.png')) diff --git a/ci/check_version_number.py b/ci/check_version_number.py index 3845cdfe27..7a3afdf5a0 100644 --- a/ci/check_version_number.py +++ b/ci/check_version_number.py @@ -1,8 +1,19 @@ """ -On release, ensure that the release tag matches the InvenTree version number! +Ensure that the release tag matches the InvenTree version number: + +master / main branch: + - version number must end with 'dev' + +stable branch: + - version number must *not* end with 'dev' + - version number cannot already exist as a release tag + +tagged branch: + - version number must match tag being built + - version number cannot already exist as a release tag + """ -import argparse import os import re import sys @@ -11,6 +22,15 @@ if __name__ == '__main__': here = os.path.abspath(os.path.dirname(__file__)) + # GITHUB_REF_TYPE may be either 'branch' or 'tag' + GITHUB_REF_TYPE = os.environ['GITHUB_REF_TYPE'] + + # GITHUB_REF may be either 'refs/heads/' or 'refs/heads/' + GITHUB_REF = os.environ['GITHUB_REF'] + + # GITHUB_BASE_REF is the base branch e.g. 'master' or 'stable' + GITHUB_BASE_REF = os.environ['GITHUB_BASE_REF'] + version_file = os.path.join(here, '..', 'InvenTree', 'InvenTree', 'version.py') version = None @@ -30,66 +50,65 @@ if __name__ == '__main__': print(f"InvenTree Version: '{version}'") - parser = argparse.ArgumentParser() - parser.add_argument('-t', '--tag', help='Compare against specified version tag', action='store') - parser.add_argument('-r', '--release', help='Check that this is a release version', action='store_true') - parser.add_argument('-d', '--dev', help='Check that this is a development version', action='store_true') - parser.add_argument('-b', '--branch', help='Check against a particular branch', action='store') + # Determine which docker tag we are going to use + docker_tag = None - args = parser.parse_args() - - if args.branch: - """ - Version number requirement depends on format of branch - - 'master': development branch - 'stable': release branch - """ - - print(f"Checking version number for branch '{args.branch}'") - - if args.branch == 'master': - print("- This is a development branch") - args.dev = True - elif args.branch == 'stable': - print("- This is a stable release branch") - args.release = True - - if args.dev: - """ - Check that the current verrsion number matches the "development" format - e.g. "0.5 dev" - """ - - print("Checking development branch") - - pattern = r"^\d+(\.\d+)+ dev$" - - result = re.match(pattern, version) - - if result is None: - print(f"Version number '{version}' does not match required pattern for development branch") - sys.exit(1) - - elif args.release: - """ - Check that the current version number matches the "release" format - e.g. "0.5.1" - """ - - print("Checking release branch") + if GITHUB_BASE_REF == 'stable' and GITHUB_REF_TYPE == 'branch': + print("Checking requirements for 'stable' release") pattern = r"^\d+(\.\d+)+$" - result = re.match(pattern, version) if result is None: print(f"Version number '{version}' does not match required pattern for stable branch") sys.exit(1) + else: + print(f"Version number '{version}' matches stable branch") - if args.tag: - if args.tag != version: - print(f"Release tag '{args.tag}' does not match INVENTREE_SW_VERSION '{version}'") + docker_tag = 'stable' + + elif GITHUB_BASE_REF in ['master', 'main'] and GITHUB_REF_TYPE == 'branch': + print("Checking requirements for main development branch:") + + pattern = r"^\d+(\.\d+)+ dev$" + result = re.match(pattern, version) + + if result is None: + print(f"Version number '{version}' does not match required pattern for development branch") sys.exit(1) + else: + print(f"Version number '{version}' matches development branch") -sys.exit(0) + docker_tag = 'latest' + + elif GITHUB_REF_TYPE == 'tag': + # GITHUB_REF should be of th eform /refs/heads/ + version_tag = GITHUB_REF.split('/')[-1] + print(f"Checking requirements for tagged release - '{version_tag}'") + + if version_tag != version: + print(f"Version number '{version}' does not match tag '{version_tag}'") + sys.exit + + # TODO: Check if there is already a release with this tag! + + docker_tag = version_tag + + else: + print("Unsupported branch / version combination:") + print(f"InvenTree Version: {version}") + print("GITHUB_REF_TYPE:", GITHUB_REF_TYPE) + print("GITHUB_REF:", GITHUB_REF) + print("GITHUB_BASE_REF:", GITHUB_BASE_REF) + sys.exit(1) + + if docker_tag is None: + print("Docker tag could not be determined") + sys.exit(1) + + print(f"Version check passed for '{version}'!") + print(f"Docker tag: '{docker_tag}'") + + # Ref: https://getridbug.com/python/how-to-set-environment-variables-in-github-actions-using-python/ + with open(os.getenv('GITHUB_ENV'), 'a') as env_file: + env_file.write(f"docker_tag={docker_tag}\n") diff --git a/docker/docker-compose.yml b/docker-compose.yml similarity index 98% rename from docker/docker-compose.yml rename to docker-compose.yml index e8bb12c44a..baba646883 100644 --- a/docker/docker-compose.yml +++ b/docker-compose.yml @@ -101,4 +101,4 @@ volumes: o: bind # This directory specified where InvenTree source code is stored "outside" the docker containers # By default, this directory is one level above the "docker" directory - device: ${INVENTREE_EXT_VOLUME:-../} + device: ${INVENTREE_EXT_VOLUME:-./} diff --git a/docker/init.sh b/docker/init.sh index 088dd68e89..47f05afeb0 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # exit when any command fails set -e diff --git a/docker/production/.env b/docker/production/.env index 220952bf23..9bf801dba5 100644 --- a/docker/production/.env +++ b/docker/production/.env @@ -16,6 +16,12 @@ INVENTREE_WEB_PORT=1337 INVENTREE_DEBUG=False INVENTREE_LOG_LEVEL=WARNING +# InvenTree admin account details +# Un-comment (and complete) these lines to auto-create an admin acount +#INVENTREE_ADMIN_USER= +#INVENTREE_ADMIN_PASSWORD= +#INVENTREE_ADMIN_EMAIL= + # Database configuration options # Note: The example setup is for a PostgreSQL database INVENTREE_DB_ENGINE=postgresql diff --git a/requirements.txt b/requirements.txt index 822d40fc54..9b857e72ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,16 +29,16 @@ django-sslserver==0.22 # Secure HTTP development server django-stdimage==5.1.1 # Advanced ImageField management django-test-migrations==1.1.0 # Unit testing for database migrations django-user-sessions==1.7.1 # user sessions in DB -django-weasyprint==1.0.1 # django weasyprint integration +django-weasyprint==2.1.0 # django weasyprint integration djangorestframework==3.12.4 # DRF framework django-xforwardedfor-middleware==2.0 # IP forwarding metadata flake8==3.8.3 # PEP checking flake8-docstrings==1.6.0 # docstring format testing gunicorn>=20.1.0 # Gunicorn web server importlib_metadata # Backport for importlib.metadata -inventree # Install the latest version of the InvenTree API python library isort==5.10.1 # DEV: python import sorting markdown==3.3.4 # Force particular version of markdown +pdf2image==1.16.0 # PDF to image conversion pep8-naming==0.11.1 # PEP naming convention extension pre-commit==2.19.0 # Git pre-commit pillow==9.1.0 # Image manipulation @@ -48,4 +48,4 @@ python-barcode[images]==0.13.1 # Barcode generator qrcode[pil]==6.1 # QR code generator rapidfuzz==0.7.6 # Fuzzy string matching tablib[xls,xlsx,yaml] # Support for XLS and XLSX formats -weasyprint==52.5 # PDF generation library (Note: in the future need to update to 53) +weasyprint==55.0 # PDF generation library diff --git a/tasks.py b/tasks.py index 9fa55c4513..4ecebfc8e8 100644 --- a/tasks.py +++ b/tasks.py @@ -82,7 +82,7 @@ def plugins(c): print(f"Installing plugin packages from '{plugin_file}'") # Install the plugins - c.run(f"pip3 install -U -r '{plugin_file}'") + c.run(f"pip3 install --disable-pip-version-check -U -r '{plugin_file}'") @task(post=[plugins]) @@ -94,7 +94,7 @@ def install(c): print("Installing required python packages from 'requirements.txt'") # Install required Python packages with PIP - c.run('pip3 install -U -r requirements.txt') + c.run('pip3 install --no-cache-dir --disable-pip-version-check -U -r requirements.txt') @task