From 4d8836378b0d55eeeb5af6c63d6775afa375f7d8 Mon Sep 17 00:00:00 2001 From: Matthias Mair Date: Sat, 28 May 2022 02:38:12 +0200 Subject: [PATCH] CI updates (#3087) * Add pre-commit to the stack * exclude static * Add locales to excludes * fix style errors * rename pipeline steps * also wait on precommit * make template matching simpler * Use the same code for python setup everywhere * use step and cache for python setup * move regular settings up into general envs * just use full update * Use invoke instead of static references * make setup actions more similar * use python3 * refactor names to be similar * fix runner version * fix references * remove incidential change * use matrix for os * Github can't do this right now * ignore docstyle errors * Add seperate docstring test * update flake call * do not fail on docstring * refactor setup into workflow * update reference * switch to action * resturcture * add bash statements * remove os from cache * update input checks * make code cleaner * fix boolean * no relative paths * install wheel by python * switch to install * revert back to simple wheel * refactor import export tests * move setup keys back to not disturbe tests * remove docstyle till that is fixed * update references * continue on error * use relativ action references * Change step / job docstrings --- .github/actions/migration/action.yaml | 17 ++ .github/actions/setup/action.yaml | 82 ++++++ .github/workflows/qc_checks.yaml | 267 +++++++----------- .pre-commit-config.yaml | 5 + InvenTree/order/models.py | 3 +- .../stock/templates/stock/item_base.html | 4 +- InvenTree/templates/js/translated/build.js | 4 +- InvenTree/templates/js/translated/order.js | 14 +- InvenTree/templates/js/translated/tables.js | 10 +- tasks.py | 6 +- 10 files changed, 221 insertions(+), 191 deletions(-) create mode 100644 .github/actions/migration/action.yaml create mode 100644 .github/actions/setup/action.yaml diff --git a/.github/actions/migration/action.yaml b/.github/actions/migration/action.yaml new file mode 100644 index 0000000000..3270a9e55b --- /dev/null +++ b/.github/actions/migration/action.yaml @@ -0,0 +1,17 @@ +name: 'Migration test' +description: 'Run migration test sequenze' +author: 'inventree' + +runs: + using: 'composite' + steps: + - name: Data Import Export + shell: bash + run: | + invoke migrate + invoke import-fixtures + invoke export-records -f data.json + python3 ./InvenTree/manage.py flush --noinput + invoke migrate + invoke import-records -f data.json + invoke import-records -f data.json diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml new file mode 100644 index 0000000000..1a954349ef --- /dev/null +++ b/.github/actions/setup/action.yaml @@ -0,0 +1,82 @@ +name: 'Setup Enviroment' +description: 'Setup the enviroment for general InvenTree tests' +author: 'inventree' +inputs: + python: + required: false + description: 'Install python.' + default: 'true' + npm: + required: false + description: 'Install npm.' + default: 'false' + + install: + required: false + description: 'Install the InvenTree requirements?' + default: 'false' + update: + required: false + description: 'Should a full update cycle be run?' + default: 'false' + + apt-dependency: + required: false + description: 'Extra APT package for install.' + pip-dependency: + required: false + description: 'Extra python package for install.' + +runs: + using: 'composite' + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + # Python installs + - name: Set up Python ${{ env.python_version }} + if: ${{ inputs.python == 'true' }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.python_version }} + cache: pip + - name: Install Base Python Dependencies + if: ${{ inputs.python == 'true' }} + shell: bash + run: | + python3 -m pip install -U pip + pip3 install invoke wheel + - name: Install Specific Python Dependencies + if: ${{ inputs.pip-dependency }} + shell: bash + run: pip3 install ${{ inputs.pip-dependency }} + + # NPM installs + - name: Install node.js ${{ env.node_version }} + if: ${{ inputs.npm == 'true' }} + uses: actions/setup-node@v2 + with: + node-version: ${{ env.node_version }} + cache: 'npm' + - name: Intall npm packages + if: ${{ inputs.npm == 'true' }} + shell: bash + run: npm install + + # OS installs + - name: Install OS Dependencies + if: ${{ inputs.apt-dependency }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install ${{ inputs.apt-dependency }} + + # Invoke commands + - name: Run invoke install + if: ${{ inputs.install == 'true' }} + shell: bash + run: invoke install + - name: Run invoke update + if: ${{ inputs.update == 'true' }} + shell: bash + run: invoke update diff --git a/.github/workflows/qc_checks.yaml b/.github/workflows/qc_checks.yaml index 2245a5242f..cf2700c3d3 100644 --- a/.github/workflows/qc_checks.yaml +++ b/.github/workflows/qc_checks.yaml @@ -14,120 +14,94 @@ on: env: python_version: 3.9 node_version: 16 + # The OS version must be set per job server_start_sleep: 60 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} INVENTREE_DB_ENGINE: sqlite3 INVENTREE_DB_NAME: inventree - INVENTREE_MEDIA_ROOT: ./media - INVENTREE_STATIC_ROOT: ./static - + INVENTREE_MEDIA_ROOT: ../test_inventree_media + INVENTREE_STATIC_ROOT: ../test_inventree_static jobs: pep_style: - name: PEP style (python) - runs-on: ubuntu-latest + name: Style [Python] + runs-on: ubuntu-20.04 steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Set up Python ${{ env.python_version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install deps - run: | - pip install flake8==3.8.3 - pip install pep8-naming==0.11.1 - - name: flake8 - run: | - flake8 InvenTree + install: true + - name: Run flake8 + run: flake8 InvenTree --extend-ignore=D javascript: - name: javascript template files + name: Style [JS] + runs-on: ubuntu-20.04 + needs: pep_style - runs-on: ubuntu-latest steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Install node.js ${{ env.node_version }} - uses: actions/setup-node@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - node-version: ${{ env.node_version }} - cache: 'npm' - - run: npm install - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install gettext - pip3 install invoke - invoke install - invoke static - - name: Check Templated Files + npm: true + install: true + - name: Check Templated JS Files run: | cd ci - python check_js_templates.py + python3 check_js_templates.py - name: Lint Javascript Files run: | invoke render-js-files npx eslint js_tmp/*.js html: - name: html template files + name: Style [HTML] + runs-on: ubuntu-20.04 + needs: pep_style - runs-on: ubuntu-latest steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Install node.js ${{ env.node_version }} - uses: actions/setup-node@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - node-version: ${{ env.node_version }} - cache: 'npm' - - run: npm install - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install gettext - pip3 install invoke - invoke install - invoke static + npm: true + install: true - name: Check HTML Files - run: | - npx markuplint InvenTree/build/templates/build/*.html - npx markuplint InvenTree/company/templates/company/*.html - npx markuplint InvenTree/order/templates/order/*.html - npx markuplint InvenTree/part/templates/part/*.html - npx markuplint InvenTree/stock/templates/stock/*.html - npx markuplint InvenTree/templates/*.html - npx markuplint InvenTree/templates/InvenTree/*.html - npx markuplint InvenTree/templates/InvenTree/settings/*.html + run: npx markuplint **/templates/*.html + + pre-commit: + name: Style [pre-commit] + runs-on: ubuntu-20.04 + + needs: pep_style + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ env.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.python_version }} + cache: 'pip' + - name: Run pre-commit Checks + uses: pre-commit/action@v2.0.3 python: - name: python bindings - needs: pep_style - runs-on: ubuntu-latest + name: Tests - inventree-python + runs-on: ubuntu-20.04 + + needs: pre-commit env: wrapper_name: inventree-python INVENTREE_DB_ENGINE: django.db.backends.sqlite3 INVENTREE_DB_NAME: ../inventree_unit_test_db.sqlite3 - INVENTREE_MEDIA_ROOT: ../test_inventree_media - INVENTREE_STATIC_ROOT: ../test_inventree_static INVENTREE_ADMIN_USER: testuser INVENTREE_ADMIN_PASSWORD: testpassword INVENTREE_ADMIN_EMAIL: test@test.com @@ -136,34 +110,32 @@ jobs: INVENTREE_PYTHON_TEST_PASSWORD: testpassword steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Install InvenTree - run: | - sudo apt-get update - sudo apt-get install python3-dev python3-pip python3-venv - pip3 install invoke - invoke install - invoke migrate - - name: Download Python Code - run: | - git clone --depth 1 https://github.com/inventree/${{ env.wrapper_name }} ./${{ env.wrapper_name }} - - name: Start Server + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup + with: + apt-dependency: gettext + update: true + - name: Download Python Code For `${{ env.wrapper_name }}` + run: git clone --depth 1 https://github.com/inventree/${{ env.wrapper_name }} ./${{ env.wrapper_name }} + - name: Start InvenTree Server run: | invoke delete-data -f invoke import-fixtures invoke server -a 127.0.0.1:12345 & invoke wait - - name: Run Tests + - name: Run Tests For `${{ env.wrapper_name }}` run: | cd ${{ env.wrapper_name }} invoke check-server coverage run -m unittest discover -s test/ coverage: - name: Sqlite / coverage - needs: ['javascript', 'html'] - runs-on: ubuntu-latest + name: Tests - DB [SQLite] + Coverage + runs-on: ubuntu-20.04 + + needs: ['javascript', 'html', 'pre-commit'] + continue-on-error: true # continue if a step fails so that coverage gets pushed env: INVENTREE_DB_NAME: ./inventree.sqlite @@ -171,32 +143,16 @@ jobs: INVENTREE_PLUGINS_ENABLED: true steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python ${{ env.python_version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install gettext - python -m pip install -U pip - pip3 install invoke - invoke update + apt-dependency: gettext + update: true - name: Coverage Tests - run: | - invoke coverage - - name: Data Import Export - run: | - invoke migrate - invoke import-fixtures - invoke export-records -f data.json - rm inventree.sqlite - invoke migrate - invoke import-records -f data.json - invoke import-records -f data.json + run: invoke coverage + - name: Data Export Test + uses: ./.github/actions/migration - name: Test Translations run: invoke translate - name: Check Migration Files @@ -205,9 +161,10 @@ jobs: run: coveralls postgres: - name: Postgres - needs: ['javascript', 'html'] - runs-on: ubuntu-latest + name: Tests - DB [PostgreSQL] + runs-on: ubuntu-20.04 + + needs: ['javascript', 'html', 'pre-commit'] if: github.event_name == 'push' env: @@ -235,38 +192,23 @@ jobs: - 6379:6379 steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python ${{ env.python_version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install libpq-dev gettext - python -m pip install -U pip - pip3 install invoke - pip3 install psycopg2 - pip3 install django-redis>=5.0.0 - invoke update + apt-dependency: gettext libpq-dev + pip-dependency: psycopg2 django-redis>=5.0.0 + update: true - name: Run Tests run: invoke test - - name: Data Import Export - run: | - invoke migrate - python3 ./InvenTree/manage.py flush --noinput - invoke import-fixtures - invoke export-records -f data.json - python3 ./InvenTree/manage.py flush --noinput - invoke import-records -f data.json - invoke import-records -f data.json + - name: Data Export Test + uses: ./.github/actions/migration mysql: - name: MySql - needs: ['javascript', 'html'] - runs-on: ubuntu-latest + name: Tests - DB [MySQL] + runs-on: ubuntu-20.04 + + needs: ['javascript', 'html', 'pre-commit'] if: github.event_name == 'push' env: @@ -293,29 +235,14 @@ jobs: - 3306:3306 steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python ${{ env.python_version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v1 + - name: Enviroment Setup + uses: ./.github/actions/setup with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install libmysqlclient-dev gettext - python -m pip install -U pip - pip3 install invoke - pip3 install mysqlclient - invoke update + apt-dependency: gettext libmysqlclient-dev + pip-dependency: mysqlclient + update: true - name: Run Tests run: invoke test - - name: Data Import Export - run: | - invoke migrate - python3 ./InvenTree/manage.py flush --noinput - invoke import-fixtures - invoke export-records -f data.json - python3 ./InvenTree/manage.py flush --noinput - invoke import-records -f data.json - invoke import-records -f data.json + - name: Data Export Test + uses: ./.github/actions/migration diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1dc2ba3f70..269ddddee9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,10 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +exclude: | + (?x)^( + InvenTree/InvenTree/static/.*| + InvenTree/locale/.* + )$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.2.0 diff --git a/InvenTree/order/models.py b/InvenTree/order/models.py index e4ad0b881f..3871ba38a1 100644 --- a/InvenTree/order/models.py +++ b/InvenTree/order/models.py @@ -22,8 +22,8 @@ from django.dispatch.dispatcher import receiver from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from djmoney.contrib.exchange.models import convert_money from djmoney.contrib.exchange.exceptions import MissingRate +from djmoney.contrib.exchange.models import convert_money from djmoney.money import Money from error_report.models import Error from markdownx.models import MarkdownxField @@ -43,7 +43,6 @@ from plugin.models import MetadataMixin from stock import models as stock_models from users import models as UserModels - logger = logging.getLogger('inventree') diff --git a/InvenTree/stock/templates/stock/item_base.html b/InvenTree/stock/templates/stock/item_base.html index 42c5df622d..9ae43c1c4b 100644 --- a/InvenTree/stock/templates/stock/item_base.html +++ b/InvenTree/stock/templates/stock/item_base.html @@ -156,7 +156,7 @@ {% endif %} - + {% if item.uid %} @@ -452,7 +452,7 @@ {% endif %} - + {% endblock details_right %} diff --git a/InvenTree/templates/js/translated/build.js b/InvenTree/templates/js/translated/build.js index 7d7cb59240..b17c6282a0 100644 --- a/InvenTree/templates/js/translated/build.js +++ b/InvenTree/templates/js/translated/build.js @@ -2582,7 +2582,7 @@ function loadBuildTable(table, options) { loaded_calendar = true; var el = document.getElementById('build-order-calendar'); - + calendar = new FullCalendar.Calendar(el, { initialView: 'dayGridMonth', nowIndicator: true, @@ -2592,7 +2592,7 @@ function loadBuildTable(table, options) { buildEvents(calendar); } }); - + calendar.render(); } else { calendar.render(); diff --git a/InvenTree/templates/js/translated/order.js b/InvenTree/templates/js/translated/order.js index 4b39f11ee8..9b4903b22b 100644 --- a/InvenTree/templates/js/translated/order.js +++ b/InvenTree/templates/js/translated/order.js @@ -1522,7 +1522,7 @@ function loadPurchaseOrderTable(table, options) { setupFilterList('purchaseorder', $(table), target, {download: true}); var display_mode = inventreeLoad('purchaseorder-table-display-mode', 'list'); - + // Function for rendering PurchaseOrder calendar display function buildEvents(calendar) { @@ -1538,7 +1538,7 @@ function loadPurchaseOrderTable(table, options) { filters.supplier_detail = true; filters.min_date = start; filters.max_date = end; - + // Request purchase orders from the server within specified date range inventreeGet( '{% url "api-po-list" %}', @@ -1718,7 +1718,7 @@ function loadPurchaseOrderTable(table, options) { if (display_mode == 'calendar') { var el = document.getElementById('purchase-order-calendar'); - + calendar = new FullCalendar.Calendar(el, { initialView: 'dayGridMonth', nowIndicator: true, @@ -1728,7 +1728,7 @@ function loadPurchaseOrderTable(table, options) { buildEvents(calendar); } }); - + calendar.render(); } } @@ -2409,7 +2409,7 @@ function loadSalesOrderTable(table, options) { if (display_mode == 'calendar') { var el = document.getElementById('purchase-order-calendar'); - + calendar = new FullCalendar.Calendar(el, { initialView: 'dayGridMonth', nowIndicator: true, @@ -2419,7 +2419,7 @@ function loadSalesOrderTable(table, options) { buildEvents(calendar); } }); - + calendar.render(); } }, @@ -2881,7 +2881,7 @@ function allocateStockToSalesOrder(order_id, line_items, options={}) { fields.reference.value = ref; fields.reference.prefix = global_settings.SALESORDER_REFERENCE_PREFIX + options.reference; - + return fields; } } diff --git a/InvenTree/templates/js/translated/tables.js b/InvenTree/templates/js/translated/tables.js index 7525620bb3..fcbaba7336 100644 --- a/InvenTree/templates/js/translated/tables.js +++ b/InvenTree/templates/js/translated/tables.js @@ -27,16 +27,16 @@ function reloadtable(table) { /* * Construct a set of extra buttons to display against a list of orders, * allowing the orders to be displayed in various 'view' modes: - * + * * - Calendar view * - List view * - Tree view - * + * * Options: * - callback: Callback function to be called when one of the buttons is pressed * - prefix: The prefix to use when saving display data to user session * - display: Which button to set as 'active' by default - * + * */ function constructOrderTableButtons(options={}) { @@ -51,10 +51,10 @@ function constructOrderTableButtons(options={}) { var idx = 0; var buttons = []; - + function buttonCallback(view_mode) { inventreeSave(key, view_mode); - + if (options.callback) { options.callback(view_mode); } diff --git a/tasks.py b/tasks.py index a5d1ac0e5b..9fa55c4513 100644 --- a/tasks.py +++ b/tasks.py @@ -554,9 +554,9 @@ def test_translations(c): # complie regex reg = re.compile( - r"[a-zA-Z0-9]{1}"+ # match any single letter and number - r"(?![^{\(\<]*[}\)\>])"+ # that is not inside curly brackets, brackets or a tag - r"(?])" + # that is not inside curly brackets, brackets or a tag # noqa: W504 + r"(?