CI updates (#3087)

* Add pre-commit to the stack

* exclude static

* Add locales to excludes

* fix style errors

* rename pipeline steps

* also wait on precommit

* make template matching simpler

* Use the same code for python setup everywhere

* use step and cache for python setup

* move regular settings up into general envs

* just use full update

* Use invoke instead of static references

* make setup actions more similar

* use python3

* refactor names to be similar

* fix runner version

* fix references

* remove incidential change

* use matrix for os

* Github can't do this right now

* ignore docstyle errors

* Add seperate docstring test

* update flake call

* do not fail on docstring

* refactor setup into workflow

* update reference

* switch to action

* resturcture

* add bash statements

* remove os from cache

* update input checks

* make code cleaner

* fix boolean

* no relative paths

* install wheel by python

* switch to install

* revert back to simple wheel

* refactor import export tests

* move setup keys back to not disturbe tests

* remove docstyle till that is fixed

* update references

* continue on error

* use relativ action references

* Change step / job docstrings

.github/actions/migration/action.yaml

@@ -0,0 +1,17 @@
+name: 'Migration test'
+description: 'Run migration test sequenze'
+author: 'inventree'
+
+runs:
+    using: 'composite'
+    steps:
+      - name: Data Import Export
+        shell: bash
+        run: |
+          invoke migrate
+          invoke import-fixtures
+          invoke export-records -f data.json
+          python3 ./InvenTree/manage.py flush --noinput
+          invoke migrate
+          invoke import-records -f data.json
+          invoke import-records -f data.json

.github/actions/setup/action.yaml

@@ -0,0 +1,82 @@
+name: 'Setup Enviroment'
+description: 'Setup the enviroment for general InvenTree tests'
+author: 'inventree'
+inputs:
+    python:
+        required: false
+        description: 'Install python.'
+        default: 'true'
+    npm:
+        required: false
+        description: 'Install npm.'
+        default: 'false'
+
+    install:
+        required: false
+        description: 'Install the InvenTree requirements?'
+        default: 'false'
+    update:
+        required: false
+        description: 'Should a full update cycle be run?'
+        default: 'false'
+
+    apt-dependency:
+        required: false
+        description: 'Extra APT package for install.'
+    pip-dependency:
+        required: false
+        description: 'Extra python package for install.'
+
+runs:
+    using: 'composite'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      # Python installs
+      - name: Set up Python ${{ env.python_version }}
+        if: ${{ inputs.python == 'true' }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python_version }}
+          cache: pip
+      - name: Install Base Python Dependencies
+        if: ${{ inputs.python == 'true' }}
+        shell: bash
+        run: |
+          python3 -m pip install -U pip
+          pip3 install invoke wheel
+      - name: Install Specific Python Dependencies
+        if: ${{ inputs.pip-dependency }}
+        shell: bash
+        run: pip3 install ${{ inputs.pip-dependency }}
+
+      # NPM installs
+      - name: Install node.js ${{ env.node_version }}
+        if: ${{ inputs.npm == 'true' }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ env.node_version }}
+          cache: 'npm'
+      - name: Intall npm packages
+        if: ${{ inputs.npm == 'true' }}
+        shell: bash
+        run: npm install
+
+      # OS installs
+      - name: Install OS Dependencies
+        if: ${{ inputs.apt-dependency }}
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install ${{ inputs.apt-dependency }}
+
+      # Invoke commands
+      - name: Run invoke install
+        if: ${{ inputs.install == 'true' }}
+        shell: bash
+        run: invoke install
+      - name: Run invoke update
+        if: ${{ inputs.update == 'true' }}
+        shell: bash
+        run: invoke update

.github/workflows/qc_checks.yaml

@@ -14,120 +14,94 @@ on:
 env:
   python_version: 3.9
   node_version: 16
+  # The OS version must be set per job
 
   server_start_sleep: 60
 
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   INVENTREE_DB_ENGINE: sqlite3
   INVENTREE_DB_NAME: inventree
-  INVENTREE_MEDIA_ROOT: ./media
+  INVENTREE_MEDIA_ROOT: ../test_inventree_media
-  INVENTREE_STATIC_ROOT: ./static
+  INVENTREE_STATIC_ROOT: ../test_inventree_static
-
 
 jobs:
   pep_style:
-    name: PEP style (python)
+    name: Style [Python]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
-      - name: Checkout code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Set up Python ${{ env.python_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-python@v2
         with:
-          python-version: ${{ env.python_version }}
+          install: true
-          cache: 'pip'
+      - name: Run flake8
-      - name: Install deps
+        run: flake8 InvenTree --extend-ignore=D
-        run: |
-          pip install flake8==3.8.3
-          pip install pep8-naming==0.11.1
-      - name: flake8
-        run: |
-          flake8 InvenTree
 
   javascript:
-    name: javascript template files
+    name: Style [JS]
+    runs-on: ubuntu-20.04
+
     needs: pep_style
-    runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Install node.js ${{ env.node_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-node@v2
         with:
-          node-version: ${{ env.node_version }}
+          npm: true
-          cache: 'npm'
+          install: true
-      - run: npm install
+      - name: Check Templated JS Files
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ env.python_version }}
-          cache: 'pip'
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install gettext
-          pip3 install invoke
-          invoke install
-          invoke static
-      - name: Check Templated Files
         run: |
           cd ci
-          python check_js_templates.py
+          python3 check_js_templates.py
       - name: Lint Javascript Files
         run: |
           invoke render-js-files
           npx eslint js_tmp/*.js
 
   html:
-    name: html template files
+    name: Style [HTML]
+    runs-on: ubuntu-20.04
+
     needs: pep_style
-    runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Install node.js ${{ env.node_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-node@v2
         with:
-          node-version: ${{ env.node_version }}
+          npm: true
-          cache: 'npm'
+          install: true
-      - run: npm install
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ env.python_version }}
-          cache: 'pip'
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install gettext
-          pip3 install invoke
-          invoke install
-          invoke static
       - name: Check HTML Files
-        run: |
+        run: npx markuplint **/templates/*.html
-          npx markuplint InvenTree/build/templates/build/*.html
+
-          npx markuplint InvenTree/company/templates/company/*.html
+  pre-commit:
-          npx markuplint InvenTree/order/templates/order/*.html
+    name: Style [pre-commit]
-          npx markuplint InvenTree/part/templates/part/*.html
+    runs-on: ubuntu-20.04
-          npx markuplint InvenTree/stock/templates/stock/*.html
+
-          npx markuplint InvenTree/templates/*.html
+    needs: pep_style
-          npx markuplint InvenTree/templates/InvenTree/*.html
+
-          npx markuplint InvenTree/templates/InvenTree/settings/*.html
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ env.python_version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ env.python_version }}
+        cache: 'pip'
+    - name: Run pre-commit Checks
+      uses: pre-commit/action@v2.0.3
 
   python:
-    name: python bindings
+    name: Tests - inventree-python
-    needs: pep_style
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-latest
+
+    needs: pre-commit
 
     env:
       wrapper_name: inventree-python
       INVENTREE_DB_ENGINE: django.db.backends.sqlite3
       INVENTREE_DB_NAME: ../inventree_unit_test_db.sqlite3
-      INVENTREE_MEDIA_ROOT: ../test_inventree_media
-      INVENTREE_STATIC_ROOT: ../test_inventree_static
       INVENTREE_ADMIN_USER: testuser
       INVENTREE_ADMIN_PASSWORD: testpassword
       INVENTREE_ADMIN_EMAIL: test@test.com
@@ -136,34 +110,32 @@ jobs:
       INVENTREE_PYTHON_TEST_PASSWORD: testpassword
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Install InvenTree
+        uses: ./.github/actions/setup
-        run: |
+        with:
-          sudo apt-get update
+          apt-dependency: gettext
-          sudo apt-get install python3-dev python3-pip python3-venv
+          update: true
-          pip3 install invoke
+      - name: Download Python Code For `${{ env.wrapper_name }}`
-          invoke install
+        run: git clone --depth 1 https://github.com/inventree/${{ env.wrapper_name }} ./${{ env.wrapper_name }}
-          invoke migrate
+      - name: Start InvenTree Server
-      - name: Download Python Code
-        run: |
-          git clone --depth 1 https://github.com/inventree/${{ env.wrapper_name }} ./${{ env.wrapper_name }}
-      - name: Start Server
         run: |
           invoke delete-data -f
           invoke import-fixtures
           invoke server -a 127.0.0.1:12345 &
           invoke wait
-      - name: Run Tests
+      - name: Run Tests For `${{ env.wrapper_name }}`
         run: |
           cd ${{ env.wrapper_name }}
           invoke check-server
           coverage run -m unittest discover -s test/
 
   coverage:
-    name: Sqlite / coverage
+    name: Tests - DB [SQLite] + Coverage
-    needs: ['javascript', 'html']
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-latest
+
+    needs: ['javascript', 'html', 'pre-commit']
+    continue-on-error: true   # continue if a step fails so that coverage gets pushed
 
     env:
       INVENTREE_DB_NAME: ./inventree.sqlite
@@ -171,32 +143,16 @@ jobs:
       INVENTREE_PLUGINS_ENABLED: true
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Setup Python ${{ env.python_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-python@v2
         with:
-          python-version: ${{ env.python_version }}
+          apt-dependency: gettext
-          cache: 'pip'
+          update: true
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install gettext
-          python -m pip install -U pip
-          pip3 install invoke
-          invoke update
       - name: Coverage Tests
-        run: |
+        run: invoke coverage
-          invoke coverage
+      - name: Data Export Test
-      - name: Data Import Export
+        uses: ./.github/actions/migration
-        run: |
-          invoke migrate
-          invoke import-fixtures
-          invoke export-records -f data.json
-          rm inventree.sqlite
-          invoke migrate
-          invoke import-records -f data.json
-          invoke import-records -f data.json
       - name: Test Translations
         run: invoke translate
       - name: Check Migration Files
@@ -205,9 +161,10 @@ jobs:
         run: coveralls
 
   postgres:
-    name: Postgres
+    name: Tests - DB [PostgreSQL]
-    needs: ['javascript', 'html']
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-latest
+
+    needs: ['javascript', 'html', 'pre-commit']
     if: github.event_name == 'push'
 
     env:
@@ -235,38 +192,23 @@ jobs:
           - 6379:6379
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Setup Python ${{ env.python_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-python@v2
         with:
-          python-version: ${{ env.python_version }}
+          apt-dependency: gettext libpq-dev
-          cache: 'pip'
+          pip-dependency: psycopg2 django-redis>=5.0.0
-      - name: Install Dependencies
+          update: true
-        run: |
-          sudo apt-get update
-          sudo apt-get install libpq-dev gettext
-          python -m pip install -U pip
-          pip3 install invoke
-          pip3 install psycopg2
-          pip3 install django-redis>=5.0.0
-          invoke update
       - name: Run Tests
         run: invoke test
-      - name: Data Import Export
+      - name: Data Export Test
-        run: |
+        uses: ./.github/actions/migration
-          invoke migrate
-          python3 ./InvenTree/manage.py flush --noinput
-          invoke import-fixtures
-          invoke export-records -f data.json
-          python3 ./InvenTree/manage.py flush --noinput
-          invoke import-records -f data.json
-          invoke import-records -f data.json
 
   mysql:
-    name: MySql
+    name: Tests - DB [MySQL]
-    needs: ['javascript', 'html']
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-latest
+
+    needs: ['javascript', 'html', 'pre-commit']
     if: github.event_name == 'push'
 
     env:
@@ -293,29 +235,14 @@ jobs:
           - 3306:3306
 
     steps:
-      - name: Checkout Code
+      - uses: actions/checkout@v1
-        uses: actions/checkout@v2
+      - name: Enviroment Setup
-      - name: Setup Python ${{ env.python_version }}
+        uses: ./.github/actions/setup
-        uses: actions/setup-python@v2
         with:
-          python-version: ${{ env.python_version }}
+          apt-dependency: gettext libmysqlclient-dev
-          cache: 'pip'
+          pip-dependency: mysqlclient
-      - name: Install Dependencies
+          update: true
-        run: |
-          sudo apt-get update
-          sudo apt-get install libmysqlclient-dev gettext
-          python -m pip install -U pip
-          pip3 install invoke
-          pip3 install mysqlclient
-          invoke update
       - name: Run Tests
         run: invoke test
-      - name: Data Import Export
+      - name: Data Export Test
-        run: |
+        uses: ./.github/actions/migration
-          invoke migrate
-          python3 ./InvenTree/manage.py flush --noinput
-          invoke import-fixtures
-          invoke export-records -f data.json
-          python3 ./InvenTree/manage.py flush --noinput
-          invoke import-records -f data.json
-          invoke import-records -f data.json

.pre-commit-config.yaml

@@ -1,5 +1,10 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+exclude: |
+    (?x)^(
+        InvenTree/InvenTree/static/.*|
+        InvenTree/locale/.*
+    )$
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0

InvenTree/order/models.py

@@ -22,8 +22,8 @@ from django.dispatch.dispatcher import receiver
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
-from djmoney.contrib.exchange.models import convert_money
 from djmoney.contrib.exchange.exceptions import MissingRate
+from djmoney.contrib.exchange.models import convert_money
 from djmoney.money import Money
 from error_report.models import Error
 from markdownx.models import MarkdownxField
@@ -43,7 +43,6 @@ from plugin.models import MetadataMixin
 from stock import models as stock_models
 from users import models as UserModels
 
-
 logger = logging.getLogger('inventree')
 
 

InvenTree/stock/templates/stock/item_base.html

@@ -156,7 +156,7 @@
             {% endif %}
         </td>
     </tr>
-    
+
     {% if item.uid %}
     <tr>
         <td><span class='fas fa-barcode'></span></td>
@@ -452,7 +452,7 @@
         {% endif %}
     </tr>
 
-    
+
 </table>
 {% endblock details_right %}
 

InvenTree/templates/js/translated/build.js

@@ -2582,7 +2582,7 @@ function loadBuildTable(table, options) {
                     loaded_calendar = true;
 
                     var el = document.getElementById('build-order-calendar');
-    
+
                     calendar = new FullCalendar.Calendar(el, {
                         initialView: 'dayGridMonth',
                         nowIndicator: true,
@@ -2592,7 +2592,7 @@ function loadBuildTable(table, options) {
                             buildEvents(calendar);
                         }
                     });
-            
+
                     calendar.render();
                 } else {
                     calendar.render();

InvenTree/templates/js/translated/order.js

@@ -1522,7 +1522,7 @@ function loadPurchaseOrderTable(table, options) {
     setupFilterList('purchaseorder', $(table), target, {download: true});
 
     var display_mode = inventreeLoad('purchaseorder-table-display-mode', 'list');
-    
+
     // Function for rendering PurchaseOrder calendar display
     function buildEvents(calendar) {
 
@@ -1538,7 +1538,7 @@ function loadPurchaseOrderTable(table, options) {
         filters.supplier_detail = true;
         filters.min_date = start;
         filters.max_date = end;
-        
+
         // Request purchase orders from the server within specified date range
         inventreeGet(
             '{% url "api-po-list" %}',
@@ -1718,7 +1718,7 @@ function loadPurchaseOrderTable(table, options) {
 
             if (display_mode == 'calendar') {
                 var el = document.getElementById('purchase-order-calendar');
-    
+
                 calendar = new FullCalendar.Calendar(el, {
                     initialView: 'dayGridMonth',
                     nowIndicator: true,
@@ -1728,7 +1728,7 @@ function loadPurchaseOrderTable(table, options) {
                         buildEvents(calendar);
                     }
                 });
-        
+
                 calendar.render();
             }
         }
@@ -2409,7 +2409,7 @@ function loadSalesOrderTable(table, options) {
 
             if (display_mode == 'calendar') {
                 var el = document.getElementById('purchase-order-calendar');
-    
+
                 calendar = new FullCalendar.Calendar(el, {
                     initialView: 'dayGridMonth',
                     nowIndicator: true,
@@ -2419,7 +2419,7 @@ function loadSalesOrderTable(table, options) {
                         buildEvents(calendar);
                     }
                 });
-        
+
                 calendar.render();
             }
         },
@@ -2881,7 +2881,7 @@ function allocateStockToSalesOrder(order_id, line_items, options={}) {
 
                         fields.reference.value = ref;
                         fields.reference.prefix = global_settings.SALESORDER_REFERENCE_PREFIX + options.reference;
-                        
+
                         return fields;
                     }
                 }

InvenTree/templates/js/translated/tables.js

@@ -27,16 +27,16 @@ function reloadtable(table) {
 /*
  * Construct a set of extra buttons to display against a list of orders,
  * allowing the orders to be displayed in various 'view' modes:
- * 
+ *
  * - Calendar view
  * - List view
  * - Tree view
- * 
+ *
  * Options:
  * - callback: Callback function to be called when one of the buttons is pressed
  * - prefix: The prefix to use when saving display data to user session
  * - display: Which button to set as 'active' by default
- * 
+ *
  */
 function constructOrderTableButtons(options={}) {
 
@@ -51,10 +51,10 @@ function constructOrderTableButtons(options={}) {
 
     var idx = 0;
     var buttons = [];
-    
+
     function buttonCallback(view_mode) {
         inventreeSave(key, view_mode);
-        
+
         if (options.callback) {
             options.callback(view_mode);
         }

tasks.py

@@ -554,9 +554,9 @@ def test_translations(c):
 
     # complie regex
     reg = re.compile(
-        r"[a-zA-Z0-9]{1}"+  # match any single letter and number
+        r"[a-zA-Z0-9]{1}" +  # match any single letter and number  # noqa: W504
-        r"(?![^{\(\<]*[}\)\>])"+  # that is not inside curly brackets, brackets or a tag
+        r"(?![^{\(\<]*[}\)\>])" +  # that is not inside curly brackets, brackets or a tag  # noqa: W504
-        r"(?<![^\%][^\(][)][a-z])"+  # that is not a specially formatted variable with singles
+        r"(?<![^\%][^\(][)][a-z])" +  # that is not a specially formatted variable with singles  # noqa: W504
         r"(?![^\\][\n])"  # that is not a newline
     )
     last_string = ''