Merge remote-tracking branch 'inventree/master'

.github/workflows/coverage.yaml

@@ -44,6 +44,7 @@ jobs:
           rm test_db.sqlite
           invoke migrate
           invoke import-records -f data.json
+          invoke import-records -f data.json
       - name: Test Translations 
         run: invoke translate
       - name: Check Migration Files

.github/workflows/docker_build.yaml

@@ -15,6 +15,10 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Login to Dockerhub
         uses: docker/login-action@v1
         with:
@@ -24,6 +28,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: ./docker
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
           push: true
           repository: inventree/inventree
           tags: inventree/inventree:latest

.github/workflows/docker_publish.yaml

@@ -13,6 +13,10 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: cd
         run: |
           cd docker
@@ -24,3 +28,4 @@ jobs:
           repository: inventree/inventree
           tag_with_ref: true
           dockerfile: ./Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

.github/workflows/mysql.yaml

@@ -49,3 +49,12 @@ jobs:
           invoke install
       - name: Run Tests
         run: invoke test
+      - name: Data Import Export
+        run: |
+          invoke migrate
+          python3 ./InvenTree/manage.py flush --noinput
+          invoke import-fixtures
+          invoke export-records -f data.json
+          python3 ./InvenTree/manage.py flush --noinput
+          invoke import-records -f data.json
+          invoke import-records -f data.json

.github/workflows/postgresql.yaml

@@ -45,3 +45,12 @@ jobs:
           invoke install
       - name: Run Tests
         run: invoke test
+      - name: Data Import Export
+        run: |
+          invoke migrate
+          python3 ./InvenTree/manage.py flush --noinput
+          invoke import-fixtures
+          invoke export-records -f data.json
+          python3 ./InvenTree/manage.py flush --noinput
+          invoke import-records -f data.json
+          invoke import-records -f data.json

.gitignore

@@ -45,6 +45,11 @@ static_i18n
 # Local config file
 config.yaml
 
+# Default data file
+data.json
+*.json.tmp
+*.tmp.json
+
 # Key file
 secret_key.txt
 

InvenTree/InvenTree/apps.py

@@ -5,6 +5,7 @@ import logging
 from django.apps import AppConfig
 from django.core.exceptions import AppRegistryNotReady
 
+from InvenTree.ready import canAppAccessDatabase
 import InvenTree.tasks
 
 
@@ -16,7 +17,8 @@ class InvenTreeConfig(AppConfig):
 
     def ready(self):
 
-        self.start_background_tasks()
+        if canAppAccessDatabase():
+            self.start_background_tasks()
 
     def start_background_tasks(self):
 

InvenTree/InvenTree/ready.py

@@ -0,0 +1,35 @@
+import sys
+
+
+def canAppAccessDatabase():
+    """
+    Returns True if the apps.py file can access database records.
+
+    There are some circumstances where we don't want the ready function in apps.py
+    to touch the database
+    """
+
+    # If any of the following management commands are being executed,
+    # prevent custom "on load" code from running!
+    excluded_commands = [
+        'flush',
+        'loaddata',
+        'dumpdata',
+        'makemirations',
+        'migrate',
+        'check',
+        'mediarestore',
+        'shell',
+        'createsuperuser',
+        'wait_for_db',
+        'prerender',
+        'collectstatic',
+        'makemessages',
+        'compilemessages',
+    ]
+
+    for cmd in excluded_commands:
+        if cmd in sys.argv:
+            return False
+
+    return True

InvenTree/InvenTree/settings.py

@@ -17,7 +17,6 @@ import random
 import string
 import shutil
 import sys
-import tempfile
 from datetime import datetime
 
 import yaml
@@ -250,7 +249,6 @@ INSTALLED_APPS = [
 
     # Third part add-ons
     'django_filters',                       # Extended filter functionality
-    'dbbackup',                             # Database backup / restore
     'rest_framework',                       # DRF (Django Rest Framework)
     'rest_framework.authtoken',             # Token authentication for API
     'corsheaders',                          # Cross-origin Resource Sharing for DRF
@@ -586,17 +584,6 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
 # Use database transactions when importing / exporting data
 IMPORT_EXPORT_USE_TRANSACTIONS = True
 
-BACKUP_DIR = get_setting(
-    'INVENTREE_BACKUP_DIR',
-    CONFIG.get('backup_dir', tempfile.gettempdir()),
-)
-
-# Settings for dbbsettings app
-DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage'
-DBBACKUP_STORAGE_OPTIONS = {
-    'location': BACKUP_DIR,
-}
-
 # Internal IP addresses allowed to see the debug toolbar
 INTERNAL_IPS = [
     '127.0.0.1',

InvenTree/company/apps.py

@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 import os
 import logging
 
+from PIL import UnidentifiedImageError
+
 from django.apps import AppConfig
 from django.db.utils import OperationalError, ProgrammingError
 from django.conf import settings
 
-from PIL import UnidentifiedImageError
+from InvenTree.ready import canAppAccessDatabase
+
 
 logger = logging.getLogger("inventree")
 
@@ -20,7 +23,8 @@ class CompanyConfig(AppConfig):
         This function is called whenever the Company app is loaded.
         """
 
-        self.generate_company_thumbs()
+        if canAppAccessDatabase():
+            self.generate_company_thumbs()
 
     def generate_company_thumbs(self):
 

InvenTree/config_template.yaml

@@ -138,12 +138,6 @@ static_root: '/home/inventree/static'
 #  - git
 #  - ssh
 
-# Backup options
-# Set the backup_dir parameter to store backup files in a specific location
-# If unspecified, the local user's temp directory will be used
-# Use environment variable INVENTREE_BACKUP_DIR
-backup_dir: '/home/inventree/data/backup/'
-
 # Permit custom authentication backends
 #authentication_backends:
 #  - 'django.contrib.auth.backends.ModelBackend'

InvenTree/label/apps.py

@@ -6,6 +6,8 @@ import hashlib
 from django.apps import AppConfig
 from django.conf import settings
 
+from InvenTree.ready import canAppAccessDatabase
+
 
 logger = logging.getLogger("inventree")
 
@@ -32,8 +34,9 @@ class LabelConfig(AppConfig):
         This function is called whenever the label app is loaded
         """
 
-        self.create_stock_item_labels()
+        if canAppAccessDatabase():
-        self.create_stock_location_labels()
+            self.create_stock_item_labels()
+            self.create_stock_location_labels()
 
     def create_stock_item_labels(self):
         """

InvenTree/part/apps.py

@@ -9,6 +9,9 @@ from django.conf import settings
 
 from PIL import UnidentifiedImageError
 
+from InvenTree.ready import canAppAccessDatabase
+
+
 logger = logging.getLogger("inventree")
 
 
@@ -20,8 +23,9 @@ class PartConfig(AppConfig):
         This function is called whenever the Part app is loaded.
         """
 
-        self.generate_part_thumbnails()
+        if canAppAccessDatabase():
-        self.update_trackable_status()
+            self.generate_part_thumbnails()
+            self.update_trackable_status()
 
     def generate_part_thumbnails(self):
         """

InvenTree/report/apps.py

@@ -5,6 +5,8 @@ import logging
 from django.apps import AppConfig
 from django.conf import settings
 
+from InvenTree.ready import canAppAccessDatabase
+
 
 logger = logging.getLogger("inventree")
 
@@ -17,8 +19,9 @@ class ReportConfig(AppConfig):
         This function is called whenever the report app is loaded
         """
 
-        self.create_default_test_reports()
+        if canAppAccessDatabase():
-        self.create_default_build_reports()
+            self.create_default_test_reports()
+            self.create_default_build_reports()
 
     def create_default_reports(self, model, reports):
         """

InvenTree/users/apps.py

@@ -5,21 +5,25 @@ from django.db.utils import OperationalError, ProgrammingError
 
 from django.apps import AppConfig
 
+from InvenTree.ready import canAppAccessDatabase
+
 
 class UsersConfig(AppConfig):
     name = 'users'
 
     def ready(self):
 
-        try:
+        if canAppAccessDatabase():
-            self.assign_permissions()
-        except (OperationalError, ProgrammingError):
-            pass
 
-        try:
+            try:
-            self.update_owners()
+                self.assign_permissions()
-        except (OperationalError, ProgrammingError):
+            except (OperationalError, ProgrammingError):
-            pass
+                pass
+
+            try:
+                self.update_owners()
+            except (OperationalError, ProgrammingError):
+                pass
 
     def assign_permissions(self):
 

InvenTree/users/fixtures/users.yaml

@@ -0,0 +1,53 @@
+- model: auth.group
+  pk: 1
+  fields:
+    name: "Viewers"
+
+- model: auth.group
+  pk: 2
+  fields:
+    name: "Engineers"
+
+- model: auth.group
+  pk: 3
+  fields:
+    name: "Sales"
+
+- model: auth.user
+  pk: 1
+  fields:
+    username: "sue_the_superuser"
+    is_superuser: true
+
+- model: auth.user
+  pk: 2
+  fields:
+    username: "engineer_eddie"
+    groups:
+      - 2
+    is_active: true
+    is_staff: false
+    is_superuser: false
+
+- model: auth.user
+  pk: 3
+  fields:
+    username: "alanallgroup"
+    first_name: "Alan"
+    last_name: "Allgroup"
+    is_active: false
+    groups:
+      - 1
+      - 2
+      - 3
+
+- model: auth.user
+  pk: 4
+  fields:
+    username: "sam"
+    first_name: "Samuel"
+    last_name: "Salesperson"
+    groups:
+      - 3
+    is_staff: true
+    is_superuser: true

InvenTree/users/models.py

@@ -14,6 +14,8 @@ from django.db.models.signals import post_save, post_delete
 
 import logging
 
+from InvenTree.ready import canAppAccessDatabase
+
 
 logger = logging.getLogger("inventree")
 
@@ -270,6 +272,9 @@ def update_group_roles(group, debug=False):
 
     """
 
+    if not canAppAccessDatabase():
+        return
+
     # List of permissions already associated with this group
     group_permissions = set()
 

README.md

@@ -35,10 +35,12 @@ InvenTree is supported by a [companion mobile app](https://inventree.readthedocs
 # Translation
 
 ![de translation](https://img.shields.io/badge/dynamic/json?color=blue&label=de&style=flat&query=%24.progress.0.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
+![es-ES translation](https://img.shields.io/badge/dynamic/json?color=blue&label=es-ES&style=flat&query=%24.progress.1.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 ![fr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=fr&style=flat&query=%24.progress.3.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 ![it translation](https://img.shields.io/badge/dynamic/json?color=blue&label=it&style=flat&query=%24.progress.4.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 ![pl translation](https://img.shields.io/badge/dynamic/json?color=blue&label=pl&style=flat&query=%24.progress.5.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 ![ru translation](https://img.shields.io/badge/dynamic/json?color=blue&label=ru&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
+![tr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=tr&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 ![zh-CN translation](https://img.shields.io/badge/dynamic/json?color=blue&label=zh-CN&style=flat&query=%24.progress.7.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json)
 
 Native language translation of the InvenTree web application is [community contributed via crowdin](https://crowdin.com/project/inventree). **Contributions are welcomed and encouraged**.

docker/Dockerfile

@@ -21,7 +21,6 @@ ENV INVENTREE_MNG_DIR="${INVENTREE_SRC_DIR}/InvenTree"
 ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data"
 ENV INVENTREE_STATIC_ROOT="${INVENTREE_HOME}/static"
 ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media"
-ENV INVENTREE_BACKUP_DIR="${INVENTREE_DATA_DIR}/backup"
 
 ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml"
 ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt"

docker/start_dev_server.sh

@@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then
     mkdir $INVENTREE_MEDIA_ROOT
 fi
 
-if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then
-    echo "Creating directory $INVENTREE_BACKUP_DIR"
-    mkdir $INVENTREE_BACKUP_DIR
-fi
-
 # Check if "config.yaml" has been copied into the correct location
 if test -f "$INVENTREE_CONFIG_FILE"; then
     echo "$INVENTREE_CONFIG_FILE exists - skipping"

docker/start_prod_server.sh

@@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then
     mkdir $INVENTREE_MEDIA_ROOT
 fi
 
-if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then
-    echo "Creating directory $INVENTREE_BACKUP_DIR"
-    mkdir $INVENTREE_BACKUP_DIR
-fi
-
 # Check if "config.yaml" has been copied into the correct location
 if test -f "$INVENTREE_CONFIG_FILE"; then
     echo "$INVENTREE_CONFIG_FILE exists - skipping"

requirements.txt

@@ -3,7 +3,6 @@ wheel>=0.34.2                   # Wheel
 Django==3.2                     # Django package
 pillow==8.1.1                   # Image manipulation
 djangorestframework==3.12.4     # DRF framework
-django-dbbackup==3.3.0          # Database backup / restore functionality
 django-cors-headers==3.2.0      # CORS headers extension for DRF
 django-filter==2.4.0            # Extended filtering options
 django-mptt==0.11.0             # Modified Preorder Tree Traversal

tasks.py

@@ -2,6 +2,7 @@
 
 from shutil import copyfile
 import os
+import json
 import sys
 
 try:
@@ -232,6 +233,31 @@ def coverage(c):
     # Generate coverage report
     c.run('coverage html')
 
+
+def content_excludes():
+    """
+    Returns a list of content types to exclude from import/export
+    """
+
+    excludes = [
+        "contenttypes",
+        "sessions.session",
+        "auth.permission",
+        "error_report.error",
+        "admin.logentry",
+        "django_q.schedule",
+        "django_q.task",
+        "django_q.ormq",
+    ]
+
+    output = ""
+
+    for e in excludes:
+        output += f"--exclude {e} "
+
+    return output
+
+
 @task(help={'filename': "Output filename (default = 'data.json')"})
 def export_records(c, filename='data.json'):
     """
@@ -253,10 +279,37 @@ def export_records(c, filename='data.json'):
             print("Cancelled export operation")
             sys.exit(1)
 
-    cmd = f'dumpdata --exclude contenttypes --exclude auth.permission --indent 2 --output {filename}'
+    tmpfile = f"{filename}.tmp"
 
+    cmd = f"dumpdata --indent 2 --output {tmpfile} {content_excludes()}"
+
+    # Dump data to temporary file
     manage(c, cmd, pty=True)
 
+    print("Running data post-processing step...")
+
+    # Post-process the file, to remove any "permissions" specified for a user or group
+    with open(tmpfile, "r") as f_in:
+        data = json.loads(f_in.read())
+
+    for entry in data:
+        if "model" in entry:
+            
+            # Clear out any permissions specified for a group
+            if entry["model"] == "auth.group":
+                entry["fields"]["permissions"] = []
+
+            # Clear out any permissions specified for a user
+            if entry["model"] == "auth.user":
+                entry["fields"]["user_permissions"] = []
+
+    # Write the processed data to file
+    with open(filename, "w") as f_out:
+        f_out.write(json.dumps(data, indent=2))
+
+    print("Data export completed")
+
+
 @task(help={'filename': 'Input filename'})
 def import_records(c, filename='data.json'):
     """
@@ -273,10 +326,33 @@ def import_records(c, filename='data.json'):
 
     print(f"Importing database records from '{filename}'")
 
-    cmd = f'loaddata {filename}'
+    # Pre-process the data, to remove any "permissions" specified for a user or group
+    tmpfile = f"{filename}.tmp.json"
+
+    with open(filename, "r") as f_in:
+        data = json.loads(f_in.read())
+
+    for entry in data:
+        if "model" in entry:
+            
+            # Clear out any permissions specified for a group
+            if entry["model"] == "auth.group":
+                entry["fields"]["permissions"] = []
+
+            # Clear out any permissions specified for a user
+            if entry["model"] == "auth.user":
+                entry["fields"]["user_permissions"] = []
+
+    # Write the processed data to the tmp file
+    with open(tmpfile, "w") as f_out:
+        f_out.write(json.dumps(data, indent=2))
+
+    cmd = f"loaddata {tmpfile} -i {content_excludes()}"
 
     manage(c, cmd, pty=True)
 
+    print("Data import completed")
+
 @task
 def import_fixtures(c):
     """
@@ -316,33 +392,15 @@ def import_fixtures(c):
         'location',
         'stock_tests',
         'stock',
+
+        # Users
+        'users'
     ]
 
     command = 'loaddata ' + ' '.join(fixtures)
 
     manage(c, command, pty=True)
 
-@task
-def backup(c):
-    """
-    Create a backup of database models and uploaded media files.
-
-    Backup files will be written to the 'backup_dir' file specified in 'config.yaml'
-    """
-
-    manage(c, 'dbbackup')
-    manage(c, 'mediabackup')
-
-@task
-def restore(c):
-    """
-    Restores database models and media files.
-
-    Backup files are read from the 'backup_dir' file specified in 'config.yaml'
-    """
-
-    manage(c, 'dbrestore')
-    manage(c, 'mediarestore')
 
 @task(help={'address': 'Server address:port (default=127.0.0.1:8000)'})
 def server(c, address="127.0.0.1:8000"):