diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index d903b737e4..ad5f7a841e 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -44,6 +44,7 @@ jobs: rm test_db.sqlite invoke migrate invoke import-records -f data.json + invoke import-records -f data.json - name: Test Translations run: invoke translate - name: Check Migration Files diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml index df747bc56e..89d52664e4 100644 --- a/.github/workflows/docker_build.yaml +++ b/.github/workflows/docker_build.yaml @@ -15,6 +15,10 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Login to Dockerhub uses: docker/login-action@v1 with: @@ -24,6 +28,7 @@ jobs: uses: docker/build-push-action@v2 with: context: ./docker + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true repository: inventree/inventree tags: inventree/inventree:latest diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 667b860b13..c25696d6dd 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -13,6 +13,10 @@ jobs: steps: - name: Check out repo uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: cd run: | cd docker @@ -23,4 +27,5 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} repository: inventree/inventree tag_with_ref: true - dockerfile: ./Dockerfile \ No newline at end of file + dockerfile: ./Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 diff --git a/.github/workflows/mysql.yaml b/.github/workflows/mysql.yaml index 70acad66a1..087a866fbd 100644 --- a/.github/workflows/mysql.yaml +++ b/.github/workflows/mysql.yaml @@ -48,4 +48,13 @@ jobs: pip3 install mysqlclient invoke install - name: Run Tests - run: invoke test \ No newline at end of file + run: invoke test + - name: Data Import Export + run: | + invoke migrate + python3 ./InvenTree/manage.py flush --noinput + invoke import-fixtures + invoke export-records -f data.json + python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json + invoke import-records -f data.json \ No newline at end of file diff --git a/.github/workflows/postgresql.yaml b/.github/workflows/postgresql.yaml index 76981e5a1b..ae8f52b962 100644 --- a/.github/workflows/postgresql.yaml +++ b/.github/workflows/postgresql.yaml @@ -44,4 +44,13 @@ jobs: pip3 install psycopg2 invoke install - name: Run Tests - run: invoke test \ No newline at end of file + run: invoke test + - name: Data Import Export + run: | + invoke migrate + python3 ./InvenTree/manage.py flush --noinput + invoke import-fixtures + invoke export-records -f data.json + python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json + invoke import-records -f data.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index eaa9e5574d..54ad8f07b6 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,11 @@ static_i18n # Local config file config.yaml +# Default data file +data.json +*.json.tmp +*.tmp.json + # Key file secret_key.txt diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index 03eb2bcb60..148638b5b2 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -5,6 +5,7 @@ import logging from django.apps import AppConfig from django.core.exceptions import AppRegistryNotReady +from InvenTree.ready import canAppAccessDatabase import InvenTree.tasks @@ -16,7 +17,8 @@ class InvenTreeConfig(AppConfig): def ready(self): - self.start_background_tasks() + if canAppAccessDatabase(): + self.start_background_tasks() def start_background_tasks(self): diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py new file mode 100644 index 0000000000..aa31fac947 --- /dev/null +++ b/InvenTree/InvenTree/ready.py @@ -0,0 +1,35 @@ +import sys + + +def canAppAccessDatabase(): + """ + Returns True if the apps.py file can access database records. + + There are some circumstances where we don't want the ready function in apps.py + to touch the database + """ + + # If any of the following management commands are being executed, + # prevent custom "on load" code from running! + excluded_commands = [ + 'flush', + 'loaddata', + 'dumpdata', + 'makemirations', + 'migrate', + 'check', + 'mediarestore', + 'shell', + 'createsuperuser', + 'wait_for_db', + 'prerender', + 'collectstatic', + 'makemessages', + 'compilemessages', + ] + + for cmd in excluded_commands: + if cmd in sys.argv: + return False + + return True diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 596dbd29c8..4fd8efae1a 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -17,7 +17,6 @@ import random import string import shutil import sys -import tempfile from datetime import datetime import yaml @@ -250,7 +249,6 @@ INSTALLED_APPS = [ # Third part add-ons 'django_filters', # Extended filter functionality - 'dbbackup', # Database backup / restore 'rest_framework', # DRF (Django Rest Framework) 'rest_framework.authtoken', # Token authentication for API 'corsheaders', # Cross-origin Resource Sharing for DRF @@ -586,17 +584,6 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3' # Use database transactions when importing / exporting data IMPORT_EXPORT_USE_TRANSACTIONS = True -BACKUP_DIR = get_setting( - 'INVENTREE_BACKUP_DIR', - CONFIG.get('backup_dir', tempfile.gettempdir()), -) - -# Settings for dbbsettings app -DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage' -DBBACKUP_STORAGE_OPTIONS = { - 'location': BACKUP_DIR, -} - # Internal IP addresses allowed to see the debug toolbar INTERNAL_IPS = [ '127.0.0.1', diff --git a/InvenTree/company/apps.py b/InvenTree/company/apps.py index 53e884c50f..4366f63434 100644 --- a/InvenTree/company/apps.py +++ b/InvenTree/company/apps.py @@ -3,11 +3,14 @@ from __future__ import unicode_literals import os import logging +from PIL import UnidentifiedImageError + from django.apps import AppConfig from django.db.utils import OperationalError, ProgrammingError from django.conf import settings -from PIL import UnidentifiedImageError +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -20,7 +23,8 @@ class CompanyConfig(AppConfig): This function is called whenever the Company app is loaded. """ - self.generate_company_thumbs() + if canAppAccessDatabase(): + self.generate_company_thumbs() def generate_company_thumbs(self): diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index 0a86d88827..de09e37e4f 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -138,12 +138,6 @@ static_root: '/home/inventree/static' # - git # - ssh -# Backup options -# Set the backup_dir parameter to store backup files in a specific location -# If unspecified, the local user's temp directory will be used -# Use environment variable INVENTREE_BACKUP_DIR -backup_dir: '/home/inventree/data/backup/' - # Permit custom authentication backends #authentication_backends: # - 'django.contrib.auth.backends.ModelBackend' @@ -159,4 +153,4 @@ backup_dir: '/home/inventree/data/backup/' # - 'django.contrib.auth.middleware.AuthenticationMiddleware' # - 'django.contrib.messages.middleware.MessageMiddleware' # - 'django.middleware.clickjacking.XFrameOptionsMiddleware' -# - 'InvenTree.middleware.AuthRequiredMiddleware' \ No newline at end of file +# - 'InvenTree.middleware.AuthRequiredMiddleware' diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index 4200b6e8bc..2b99921703 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -6,6 +6,8 @@ import hashlib from django.apps import AppConfig from django.conf import settings +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -32,8 +34,9 @@ class LabelConfig(AppConfig): This function is called whenever the label app is loaded """ - self.create_stock_item_labels() - self.create_stock_location_labels() + if canAppAccessDatabase(): + self.create_stock_item_labels() + self.create_stock_location_labels() def create_stock_item_labels(self): """ diff --git a/InvenTree/part/apps.py b/InvenTree/part/apps.py index 11329abdd9..531b74442a 100644 --- a/InvenTree/part/apps.py +++ b/InvenTree/part/apps.py @@ -9,6 +9,9 @@ from django.conf import settings from PIL import UnidentifiedImageError +from InvenTree.ready import canAppAccessDatabase + + logger = logging.getLogger("inventree") @@ -20,8 +23,9 @@ class PartConfig(AppConfig): This function is called whenever the Part app is loaded. """ - self.generate_part_thumbnails() - self.update_trackable_status() + if canAppAccessDatabase(): + self.generate_part_thumbnails() + self.update_trackable_status() def generate_part_thumbnails(self): """ diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index 77529263f6..43c52b1997 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -5,6 +5,8 @@ import logging from django.apps import AppConfig from django.conf import settings +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -17,8 +19,9 @@ class ReportConfig(AppConfig): This function is called whenever the report app is loaded """ - self.create_default_test_reports() - self.create_default_build_reports() + if canAppAccessDatabase(): + self.create_default_test_reports() + self.create_default_build_reports() def create_default_reports(self, model, reports): """ diff --git a/InvenTree/users/apps.py b/InvenTree/users/apps.py index 1541b1aed4..a9f671895d 100644 --- a/InvenTree/users/apps.py +++ b/InvenTree/users/apps.py @@ -5,21 +5,25 @@ from django.db.utils import OperationalError, ProgrammingError from django.apps import AppConfig +from InvenTree.ready import canAppAccessDatabase + class UsersConfig(AppConfig): name = 'users' def ready(self): - try: - self.assign_permissions() - except (OperationalError, ProgrammingError): - pass + if canAppAccessDatabase(): - try: - self.update_owners() - except (OperationalError, ProgrammingError): - pass + try: + self.assign_permissions() + except (OperationalError, ProgrammingError): + pass + + try: + self.update_owners() + except (OperationalError, ProgrammingError): + pass def assign_permissions(self): diff --git a/InvenTree/users/fixtures/users.yaml b/InvenTree/users/fixtures/users.yaml new file mode 100644 index 0000000000..5103953ca1 --- /dev/null +++ b/InvenTree/users/fixtures/users.yaml @@ -0,0 +1,53 @@ +- model: auth.group + pk: 1 + fields: + name: "Viewers" + +- model: auth.group + pk: 2 + fields: + name: "Engineers" + +- model: auth.group + pk: 3 + fields: + name: "Sales" + +- model: auth.user + pk: 1 + fields: + username: "sue_the_superuser" + is_superuser: true + +- model: auth.user + pk: 2 + fields: + username: "engineer_eddie" + groups: + - 2 + is_active: true + is_staff: false + is_superuser: false + +- model: auth.user + pk: 3 + fields: + username: "alanallgroup" + first_name: "Alan" + last_name: "Allgroup" + is_active: false + groups: + - 1 + - 2 + - 3 + +- model: auth.user + pk: 4 + fields: + username: "sam" + first_name: "Samuel" + last_name: "Salesperson" + groups: + - 3 + is_staff: true + is_superuser: true diff --git a/InvenTree/users/models.py b/InvenTree/users/models.py index 19937af190..cd222cb2a2 100644 --- a/InvenTree/users/models.py +++ b/InvenTree/users/models.py @@ -14,6 +14,8 @@ from django.db.models.signals import post_save, post_delete import logging +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -270,6 +272,9 @@ def update_group_roles(group, debug=False): """ + if not canAppAccessDatabase(): + return + # List of permissions already associated with this group group_permissions = set() diff --git a/README.md b/README.md index f8e938726d..6a665e9654 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,12 @@ InvenTree is supported by a [companion mobile app](https://inventree.readthedocs # Translation ![de translation](https://img.shields.io/badge/dynamic/json?color=blue&label=de&style=flat&query=%24.progress.0.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) +![es-ES translation](https://img.shields.io/badge/dynamic/json?color=blue&label=es-ES&style=flat&query=%24.progress.1.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![fr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=fr&style=flat&query=%24.progress.3.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![it translation](https://img.shields.io/badge/dynamic/json?color=blue&label=it&style=flat&query=%24.progress.4.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![pl translation](https://img.shields.io/badge/dynamic/json?color=blue&label=pl&style=flat&query=%24.progress.5.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![ru translation](https://img.shields.io/badge/dynamic/json?color=blue&label=ru&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) +![tr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=tr&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![zh-CN translation](https://img.shields.io/badge/dynamic/json?color=blue&label=zh-CN&style=flat&query=%24.progress.7.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) Native language translation of the InvenTree web application is [community contributed via crowdin](https://crowdin.com/project/inventree). **Contributions are welcomed and encouraged**. diff --git a/docker/Dockerfile b/docker/Dockerfile index ab4dbba6b5..c95c2867df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,6 @@ ENV INVENTREE_MNG_DIR="${INVENTREE_SRC_DIR}/InvenTree" ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data" ENV INVENTREE_STATIC_ROOT="${INVENTREE_HOME}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media" -ENV INVENTREE_BACKUP_DIR="${INVENTREE_DATA_DIR}/backup" ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt" diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh index 481da3c31a..c22805b90b 100644 --- a/docker/start_dev_server.sh +++ b/docker/start_dev_server.sh @@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then mkdir $INVENTREE_MEDIA_ROOT fi -if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then - echo "Creating directory $INVENTREE_BACKUP_DIR" - mkdir $INVENTREE_BACKUP_DIR -fi - # Check if "config.yaml" has been copied into the correct location if test -f "$INVENTREE_CONFIG_FILE"; then echo "$INVENTREE_CONFIG_FILE exists - skipping" diff --git a/docker/start_prod_server.sh b/docker/start_prod_server.sh index 811e189d13..2767e844d6 100644 --- a/docker/start_prod_server.sh +++ b/docker/start_prod_server.sh @@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then mkdir $INVENTREE_MEDIA_ROOT fi -if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then - echo "Creating directory $INVENTREE_BACKUP_DIR" - mkdir $INVENTREE_BACKUP_DIR -fi - # Check if "config.yaml" has been copied into the correct location if test -f "$INVENTREE_CONFIG_FILE"; then echo "$INVENTREE_CONFIG_FILE exists - skipping" diff --git a/requirements.txt b/requirements.txt index 1392531e29..3291574084 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ wheel>=0.34.2 # Wheel Django==3.2 # Django package pillow==8.1.1 # Image manipulation djangorestframework==3.12.4 # DRF framework -django-dbbackup==3.3.0 # Database backup / restore functionality django-cors-headers==3.2.0 # CORS headers extension for DRF django-filter==2.4.0 # Extended filtering options django-mptt==0.11.0 # Modified Preorder Tree Traversal diff --git a/tasks.py b/tasks.py index 316e9b4bfb..c4eda5e5bf 100644 --- a/tasks.py +++ b/tasks.py @@ -2,6 +2,7 @@ from shutil import copyfile import os +import json import sys try: @@ -232,6 +233,31 @@ def coverage(c): # Generate coverage report c.run('coverage html') + +def content_excludes(): + """ + Returns a list of content types to exclude from import/export + """ + + excludes = [ + "contenttypes", + "sessions.session", + "auth.permission", + "error_report.error", + "admin.logentry", + "django_q.schedule", + "django_q.task", + "django_q.ormq", + ] + + output = "" + + for e in excludes: + output += f"--exclude {e} " + + return output + + @task(help={'filename': "Output filename (default = 'data.json')"}) def export_records(c, filename='data.json'): """ @@ -253,10 +279,37 @@ def export_records(c, filename='data.json'): print("Cancelled export operation") sys.exit(1) - cmd = f'dumpdata --exclude contenttypes --exclude auth.permission --indent 2 --output {filename}' + tmpfile = f"{filename}.tmp" + cmd = f"dumpdata --indent 2 --output {tmpfile} {content_excludes()}" + + # Dump data to temporary file manage(c, cmd, pty=True) + print("Running data post-processing step...") + + # Post-process the file, to remove any "permissions" specified for a user or group + with open(tmpfile, "r") as f_in: + data = json.loads(f_in.read()) + + for entry in data: + if "model" in entry: + + # Clear out any permissions specified for a group + if entry["model"] == "auth.group": + entry["fields"]["permissions"] = [] + + # Clear out any permissions specified for a user + if entry["model"] == "auth.user": + entry["fields"]["user_permissions"] = [] + + # Write the processed data to file + with open(filename, "w") as f_out: + f_out.write(json.dumps(data, indent=2)) + + print("Data export completed") + + @task(help={'filename': 'Input filename'}) def import_records(c, filename='data.json'): """ @@ -273,10 +326,33 @@ def import_records(c, filename='data.json'): print(f"Importing database records from '{filename}'") - cmd = f'loaddata {filename}' + # Pre-process the data, to remove any "permissions" specified for a user or group + tmpfile = f"{filename}.tmp.json" + + with open(filename, "r") as f_in: + data = json.loads(f_in.read()) + + for entry in data: + if "model" in entry: + + # Clear out any permissions specified for a group + if entry["model"] == "auth.group": + entry["fields"]["permissions"] = [] + + # Clear out any permissions specified for a user + if entry["model"] == "auth.user": + entry["fields"]["user_permissions"] = [] + + # Write the processed data to the tmp file + with open(tmpfile, "w") as f_out: + f_out.write(json.dumps(data, indent=2)) + + cmd = f"loaddata {tmpfile} -i {content_excludes()}" manage(c, cmd, pty=True) + print("Data import completed") + @task def import_fixtures(c): """ @@ -316,33 +392,15 @@ def import_fixtures(c): 'location', 'stock_tests', 'stock', + + # Users + 'users' ] command = 'loaddata ' + ' '.join(fixtures) manage(c, command, pty=True) -@task -def backup(c): - """ - Create a backup of database models and uploaded media files. - - Backup files will be written to the 'backup_dir' file specified in 'config.yaml' - """ - - manage(c, 'dbbackup') - manage(c, 'mediabackup') - -@task -def restore(c): - """ - Restores database models and media files. - - Backup files are read from the 'backup_dir' file specified in 'config.yaml' - """ - - manage(c, 'dbrestore') - manage(c, 'mediarestore') @task(help={'address': 'Server address:port (default=127.0.0.1:8000)'}) def server(c, address="127.0.0.1:8000"):