From 654f3ac91b4a2f2c6cf461f50f10e11e42fcab68 Mon Sep 17 00:00:00 2001 From: 42CrMo4 <44754810+42CrMo4@users.noreply.github.com> Date: Thu, 22 Apr 2021 18:43:36 +0200 Subject: [PATCH 01/23] Docker multi-platform build --- .github/workflows/docker_build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml index df747bc56e..14ce43b776 100644 --- a/.github/workflows/docker_build.yaml +++ b/.github/workflows/docker_build.yaml @@ -15,6 +15,10 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Login to Dockerhub uses: docker/login-action@v1 with: @@ -24,6 +28,7 @@ jobs: uses: docker/build-push-action@v2 with: context: ./docker + platforms: linux/amd64,linux/arm64 push: true repository: inventree/inventree tags: inventree/inventree:latest From 2f13e5a027f42f732ee0192c88f4cba21892d209 Mon Sep 17 00:00:00 2001 From: 42CrMo4 <44754810+42CrMo4@users.noreply.github.com> Date: Thu, 22 Apr 2021 19:28:21 +0200 Subject: [PATCH 02/23] Docker multi-platform publish --- .github/workflows/docker_publish.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 667b860b13..05d8c56b4f 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -13,6 +13,10 @@ jobs: steps: - name: Check out repo uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: cd run: | cd docker @@ -23,4 +27,5 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} repository: inventree/inventree tag_with_ref: true - dockerfile: ./Dockerfile \ No newline at end of file + dockerfile: ./Dockerfile + platforms: linux/amd64,linux/arm64 From 43a36754dcc4da80ec6f7e2e17b09330179ebb8d Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 23 Apr 2021 09:53:38 +1000 Subject: [PATCH 03/23] Update README.md Added more translation badges --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f8e938726d..6a665e9654 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,12 @@ InvenTree is supported by a [companion mobile app](https://inventree.readthedocs # Translation ![de translation](https://img.shields.io/badge/dynamic/json?color=blue&label=de&style=flat&query=%24.progress.0.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) +![es-ES translation](https://img.shields.io/badge/dynamic/json?color=blue&label=es-ES&style=flat&query=%24.progress.1.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![fr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=fr&style=flat&query=%24.progress.3.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![it translation](https://img.shields.io/badge/dynamic/json?color=blue&label=it&style=flat&query=%24.progress.4.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![pl translation](https://img.shields.io/badge/dynamic/json?color=blue&label=pl&style=flat&query=%24.progress.5.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![ru translation](https://img.shields.io/badge/dynamic/json?color=blue&label=ru&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) +![tr translation](https://img.shields.io/badge/dynamic/json?color=blue&label=tr&style=flat&query=%24.progress.6.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) ![zh-CN translation](https://img.shields.io/badge/dynamic/json?color=blue&label=zh-CN&style=flat&query=%24.progress.7.data.translationProgress&url=https%3A%2F%2Fbadges.awesome-crowdin.com%2Fstats-14720186-452300.json) Native language translation of the InvenTree web application is [community contributed via crowdin](https://crowdin.com/project/inventree). **Contributions are welcomed and encouraged**. From d9cd94f13af42ae97baa8e810e3b6dc1d5179d6e Mon Sep 17 00:00:00 2001 From: 42CrMo4 <44754810+42CrMo4@users.noreply.github.com> Date: Fri, 23 Apr 2021 09:29:24 +0200 Subject: [PATCH 04/23] Docker multi-platform build amv7 --- .github/workflows/docker_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml index 14ce43b776..89d52664e4 100644 --- a/.github/workflows/docker_build.yaml +++ b/.github/workflows/docker_build.yaml @@ -28,7 +28,7 @@ jobs: uses: docker/build-push-action@v2 with: context: ./docker - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true repository: inventree/inventree tags: inventree/inventree:latest From d97816e5c955c94c9dae22a13c405fb97d138ff7 Mon Sep 17 00:00:00 2001 From: 42CrMo4 <44754810+42CrMo4@users.noreply.github.com> Date: Fri, 23 Apr 2021 09:30:23 +0200 Subject: [PATCH 05/23] Docker multi-platform publish amv7 --- .github/workflows/docker_publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index 05d8c56b4f..c25696d6dd 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -28,4 +28,4 @@ jobs: repository: inventree/inventree tag_with_ref: true dockerfile: ./Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 From de720afd0edda00fbfb95adc1030b62b9070d4d5 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:13:31 +1000 Subject: [PATCH 06/23] Exclude unwanted modules from import / export --- tasks.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 316e9b4bfb..a6304467d1 100644 --- a/tasks.py +++ b/tasks.py @@ -232,6 +232,30 @@ def coverage(c): # Generate coverage report c.run('coverage html') + +def content_excludes(): + """ + Returns a list of content types to exclude from import/export + """ + + excludes = [ + "contenttypes", + "auth.permission", + "error_report.error", + "admin.logentry", + "django_q.schedule", + "django_q.task", + "django_q.ormq", + ] + + output = "" + + for e in excludes: + output += f"--exclude {e} " + + return output + + @task(help={'filename': "Output filename (default = 'data.json')"}) def export_records(c, filename='data.json'): """ @@ -253,7 +277,7 @@ def export_records(c, filename='data.json'): print("Cancelled export operation") sys.exit(1) - cmd = f'dumpdata --exclude contenttypes --exclude auth.permission --indent 2 --output {filename}' + cmd = f"dumpdata --indent 2 --output {filename} {content_excludes()}" manage(c, cmd, pty=True) @@ -273,7 +297,7 @@ def import_records(c, filename='data.json'): print(f"Importing database records from '{filename}'") - cmd = f'loaddata {filename}' + cmd = f"loaddata {filename} -i {content_excludes()}" manage(c, cmd, pty=True) From 792ff161279d2a7eb06f181342ed18fdf07edec3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:14:13 +1000 Subject: [PATCH 07/23] Ignore data file --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index eaa9e5574d..b2bf940fad 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,9 @@ static_i18n # Local config file config.yaml +# Default data file +data.json + # Key file secret_key.txt From 4f9898fba518008e9d44e215a071f876d0818f90 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:30:31 +1000 Subject: [PATCH 08/23] Test import / export for MySQL and PostgreSQL --- .github/workflows/mysql.yaml | 9 ++++++++- .github/workflows/postgresql.yaml | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mysql.yaml b/.github/workflows/mysql.yaml index 70acad66a1..58a5c80388 100644 --- a/.github/workflows/mysql.yaml +++ b/.github/workflows/mysql.yaml @@ -48,4 +48,11 @@ jobs: pip3 install mysqlclient invoke install - name: Run Tests - run: invoke test \ No newline at end of file + run: invoke test + - name: Data Import Export + run: | + python3 ./InvenTree/manage.py flush --noinput + invoke import-fixtures + invoke export-records -f data.json + python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json \ No newline at end of file diff --git a/.github/workflows/postgresql.yaml b/.github/workflows/postgresql.yaml index 76981e5a1b..8f86d2c219 100644 --- a/.github/workflows/postgresql.yaml +++ b/.github/workflows/postgresql.yaml @@ -44,4 +44,11 @@ jobs: pip3 install psycopg2 invoke install - name: Run Tests - run: invoke test \ No newline at end of file + run: invoke test + - name: Data Import Export + run: | + python3 ./InvenTree/manage.py flush --noinput + invoke import-fixtures + invoke export-records -f data.json + python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json \ No newline at end of file From 2d7bff077397ee538bbfc5e471265cb9810132ae Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:30:56 +1000 Subject: [PATCH 09/23] Prevent auto-update of group permissions during loaddata stage --- InvenTree/users/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/InvenTree/users/models.py b/InvenTree/users/models.py index 19937af190..9a6908961b 100644 --- a/InvenTree/users/models.py +++ b/InvenTree/users/models.py @@ -13,6 +13,7 @@ from django.dispatch import receiver from django.db.models.signals import post_save, post_delete import logging +import sys logger = logging.getLogger("inventree") @@ -270,6 +271,14 @@ def update_group_roles(group, debug=False): """ + if 'loaddata' in sys.argv: + """ + In the case that we are importing records, + *do not* update group roles: + This will cause conflicts in the database! + """ + return + # List of permissions already associated with this group group_permissions = set() From 6e5fc431058c7302965bf3a1e263962c52b7e36e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:33:11 +1000 Subject: [PATCH 10/23] Prevent copying of labels and templates during data import phase --- InvenTree/label/apps.py | 13 +++++++++++-- InvenTree/report/apps.py | 8 ++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index 4200b6e8bc..c4ca408566 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -1,4 +1,5 @@ import os +import sys import shutil import logging import hashlib @@ -32,8 +33,16 @@ class LabelConfig(AppConfig): This function is called whenever the label app is loaded """ - self.create_stock_item_labels() - self.create_stock_location_labels() + if 'loaddata' in sys.argv: + """ + In the case we are importing records, + do not automatically copy labels across. + This can cause database conflicts! + """ + pass + else: + self.create_stock_item_labels() + self.create_stock_location_labels() def create_stock_item_labels(self): """ diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index 77529263f6..9709bb2309 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -1,4 +1,5 @@ import os +import sys import shutil import logging @@ -17,8 +18,11 @@ class ReportConfig(AppConfig): This function is called whenever the report app is loaded """ - self.create_default_test_reports() - self.create_default_build_reports() + if 'loaddata' in sys.argv: + pass + else: + self.create_default_test_reports() + self.create_default_build_reports() def create_default_reports(self, model, reports): """ From 06afd4d726c6aed2a8cfd5ac38c6acad443a5b68 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:52:31 +1000 Subject: [PATCH 11/23] Refactor "ready" state into a function --- InvenTree/InvenTree/apps.py | 4 +++- InvenTree/InvenTree/ready.py | 24 ++++++++++++++++++++++++ InvenTree/company/apps.py | 8 ++++++-- InvenTree/label/apps.py | 12 +++--------- InvenTree/part/apps.py | 8 ++++++-- InvenTree/report/apps.py | 7 +++---- InvenTree/users/apps.py | 20 ++++++++++++-------- InvenTree/users/models.py | 10 +++------- 8 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 InvenTree/InvenTree/ready.py diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index 03eb2bcb60..148638b5b2 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -5,6 +5,7 @@ import logging from django.apps import AppConfig from django.core.exceptions import AppRegistryNotReady +from InvenTree.ready import canAppAccessDatabase import InvenTree.tasks @@ -16,7 +17,8 @@ class InvenTreeConfig(AppConfig): def ready(self): - self.start_background_tasks() + if canAppAccessDatabase(): + self.start_background_tasks() def start_background_tasks(self): diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py new file mode 100644 index 0000000000..7d4f2d4eb8 --- /dev/null +++ b/InvenTree/InvenTree/ready.py @@ -0,0 +1,24 @@ +import sys + +def canAppAccessDatabase(): + """ + Returns True if the apps.py file can access database records. + + There are some circumstances where we don't want the ready function in apps.py + to touch the database: + + - "flush" command + - "loaddata" command + - "migrate" command + """ + + if 'flush' in sys.argv: + return False + + if 'loaddata' in sys.argv: + return False + + if 'migrate' in sys.argv: + return False + + return True diff --git a/InvenTree/company/apps.py b/InvenTree/company/apps.py index 53e884c50f..4366f63434 100644 --- a/InvenTree/company/apps.py +++ b/InvenTree/company/apps.py @@ -3,11 +3,14 @@ from __future__ import unicode_literals import os import logging +from PIL import UnidentifiedImageError + from django.apps import AppConfig from django.db.utils import OperationalError, ProgrammingError from django.conf import settings -from PIL import UnidentifiedImageError +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -20,7 +23,8 @@ class CompanyConfig(AppConfig): This function is called whenever the Company app is loaded. """ - self.generate_company_thumbs() + if canAppAccessDatabase(): + self.generate_company_thumbs() def generate_company_thumbs(self): diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index c4ca408566..2b99921703 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -1,5 +1,4 @@ import os -import sys import shutil import logging import hashlib @@ -7,6 +6,8 @@ import hashlib from django.apps import AppConfig from django.conf import settings +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -33,14 +34,7 @@ class LabelConfig(AppConfig): This function is called whenever the label app is loaded """ - if 'loaddata' in sys.argv: - """ - In the case we are importing records, - do not automatically copy labels across. - This can cause database conflicts! - """ - pass - else: + if canAppAccessDatabase(): self.create_stock_item_labels() self.create_stock_location_labels() diff --git a/InvenTree/part/apps.py b/InvenTree/part/apps.py index 11329abdd9..531b74442a 100644 --- a/InvenTree/part/apps.py +++ b/InvenTree/part/apps.py @@ -9,6 +9,9 @@ from django.conf import settings from PIL import UnidentifiedImageError +from InvenTree.ready import canAppAccessDatabase + + logger = logging.getLogger("inventree") @@ -20,8 +23,9 @@ class PartConfig(AppConfig): This function is called whenever the Part app is loaded. """ - self.generate_part_thumbnails() - self.update_trackable_status() + if canAppAccessDatabase(): + self.generate_part_thumbnails() + self.update_trackable_status() def generate_part_thumbnails(self): """ diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index 9709bb2309..43c52b1997 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -1,11 +1,12 @@ import os -import sys import shutil import logging from django.apps import AppConfig from django.conf import settings +from InvenTree.ready import canAppAccessDatabase + logger = logging.getLogger("inventree") @@ -18,9 +19,7 @@ class ReportConfig(AppConfig): This function is called whenever the report app is loaded """ - if 'loaddata' in sys.argv: - pass - else: + if canAppAccessDatabase(): self.create_default_test_reports() self.create_default_build_reports() diff --git a/InvenTree/users/apps.py b/InvenTree/users/apps.py index 1541b1aed4..a9f671895d 100644 --- a/InvenTree/users/apps.py +++ b/InvenTree/users/apps.py @@ -5,21 +5,25 @@ from django.db.utils import OperationalError, ProgrammingError from django.apps import AppConfig +from InvenTree.ready import canAppAccessDatabase + class UsersConfig(AppConfig): name = 'users' def ready(self): - try: - self.assign_permissions() - except (OperationalError, ProgrammingError): - pass + if canAppAccessDatabase(): - try: - self.update_owners() - except (OperationalError, ProgrammingError): - pass + try: + self.assign_permissions() + except (OperationalError, ProgrammingError): + pass + + try: + self.update_owners() + except (OperationalError, ProgrammingError): + pass def assign_permissions(self): diff --git a/InvenTree/users/models.py b/InvenTree/users/models.py index 9a6908961b..cd222cb2a2 100644 --- a/InvenTree/users/models.py +++ b/InvenTree/users/models.py @@ -13,7 +13,8 @@ from django.dispatch import receiver from django.db.models.signals import post_save, post_delete import logging -import sys + +from InvenTree.ready import canAppAccessDatabase logger = logging.getLogger("inventree") @@ -271,12 +272,7 @@ def update_group_roles(group, debug=False): """ - if 'loaddata' in sys.argv: - """ - In the case that we are importing records, - *do not* update group roles: - This will cause conflicts in the database! - """ + if not canAppAccessDatabase(): return # List of permissions already associated with this group From dd8c0276c741b3e1dfd43b6b213dda7e6faea675 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 10:52:47 +1000 Subject: [PATCH 12/23] Ignore session data when exporting records --- tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks.py b/tasks.py index a6304467d1..9efcc6999b 100644 --- a/tasks.py +++ b/tasks.py @@ -240,6 +240,7 @@ def content_excludes(): excludes = [ "contenttypes", + "sessions.session", "auth.permission", "error_report.error", "admin.logentry", From 368a5c3f631e97d18d9ad053fa1d1293d56186d8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:07:43 +1000 Subject: [PATCH 13/23] Gotsta create tables first ... --- .github/workflows/mysql.yaml | 1 + .github/workflows/postgresql.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/mysql.yaml b/.github/workflows/mysql.yaml index 58a5c80388..e4fd7ed81d 100644 --- a/.github/workflows/mysql.yaml +++ b/.github/workflows/mysql.yaml @@ -51,6 +51,7 @@ jobs: run: invoke test - name: Data Import Export run: | + invoke migrate python3 ./InvenTree/manage.py flush --noinput invoke import-fixtures invoke export-records -f data.json diff --git a/.github/workflows/postgresql.yaml b/.github/workflows/postgresql.yaml index 8f86d2c219..3938457722 100644 --- a/.github/workflows/postgresql.yaml +++ b/.github/workflows/postgresql.yaml @@ -47,6 +47,7 @@ jobs: run: invoke test - name: Data Import Export run: | + invoke migrate python3 ./InvenTree/manage.py flush --noinput invoke import-fixtures invoke export-records -f data.json From 8ad16ceb9caae25f984ec897fdeb4f3171955bc6 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:07:57 +1000 Subject: [PATCH 14/23] Refactorin' --- InvenTree/InvenTree/ready.py | 38 ++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index 7d4f2d4eb8..b03dc90bba 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -5,20 +5,34 @@ def canAppAccessDatabase(): Returns True if the apps.py file can access database records. There are some circumstances where we don't want the ready function in apps.py - to touch the database: - - - "flush" command - - "loaddata" command - - "migrate" command + to touch the database """ - if 'flush' in sys.argv: - return False - - if 'loaddata' in sys.argv: - return False + # If any of the following management commands are being executed, + # prevent custom "on load" code from running! + excluded_commands = [ + 'flush', + 'loaddata', + 'dumpdata', + 'makemirations', + 'migrate', + 'check', + 'dbbackup', + 'mediabackup', + 'dbrestore', + 'mediarestore', + 'shell', + 'createsuperuser', + 'wait_for_db', + 'prerender', + 'collectstatic', + 'makemessages', + 'compilemessages', + 'test', + ] - if 'migrate' in sys.argv: - return False + for cmd in excluded_commands: + if cmd in sys.argv: + return False return True From d9d6a2392d6fd435fbbde6e290302dd2b09415c8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:29:07 +1000 Subject: [PATCH 15/23] Custom post-processing step as part of export-records command: - We must delete the permission objects associated with a user or group, as auth.permissions are *NOT* included in the exported data file - The permissions will be automatically created on first run of the server --- tasks.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 9efcc6999b..7a6e952b15 100644 --- a/tasks.py +++ b/tasks.py @@ -2,6 +2,7 @@ from shutil import copyfile import os +import json import sys try: @@ -278,10 +279,33 @@ def export_records(c, filename='data.json'): print("Cancelled export operation") sys.exit(1) - cmd = f"dumpdata --indent 2 --output {filename} {content_excludes()}" + tmpfile = f"{filename}.tmp" + cmd = f"dumpdata --indent 2 --output {tmpfile} {content_excludes()}" + + # Dump data to temporary file manage(c, cmd, pty=True) + # Post-process the file, to remove any "permissions" specified for a group + with open(tmpfile, "r") as f_in: + data = json.loads(f_in.read()) + + for entry in data: + if "model" in entry: + + # Clear out any permissions specified for a group + if entry["model"] == "auth.group": + entry["fields"]["permissions"] = [] + + # Clear out any permissions specified for a user + if entry["model"] == "auth.user": + entry["fields"]["user_permissions"] = [] + + # Write the processed data to file + with open(filename, "w") as f_out: + f_out.write(json.dumps(data, indent=2)) + + @task(help={'filename': 'Input filename'}) def import_records(c, filename='data.json'): """ From e13850515d13ff67e280db5a3a2dd645ef86b02a Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:30:17 +1000 Subject: [PATCH 16/23] PEP fixes --- .gitignore | 1 + InvenTree/InvenTree/ready.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b2bf940fad..a2d5ef04b8 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ config.yaml # Default data file data.json +*.json.tmp # Key file secret_key.txt diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index b03dc90bba..ae9ddacdfc 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -1,5 +1,6 @@ import sys + def canAppAccessDatabase(): """ Returns True if the apps.py file can access database records. From f7667118385d9627035af49489a0b7c9031c280b Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:41:48 +1000 Subject: [PATCH 17/23] Add fixture for users / groups --- InvenTree/users/fixtures/users.yaml | 53 +++++++++++++++++++++++++++++ tasks.py | 5 +++ 2 files changed, 58 insertions(+) create mode 100644 InvenTree/users/fixtures/users.yaml diff --git a/InvenTree/users/fixtures/users.yaml b/InvenTree/users/fixtures/users.yaml new file mode 100644 index 0000000000..5103953ca1 --- /dev/null +++ b/InvenTree/users/fixtures/users.yaml @@ -0,0 +1,53 @@ +- model: auth.group + pk: 1 + fields: + name: "Viewers" + +- model: auth.group + pk: 2 + fields: + name: "Engineers" + +- model: auth.group + pk: 3 + fields: + name: "Sales" + +- model: auth.user + pk: 1 + fields: + username: "sue_the_superuser" + is_superuser: true + +- model: auth.user + pk: 2 + fields: + username: "engineer_eddie" + groups: + - 2 + is_active: true + is_staff: false + is_superuser: false + +- model: auth.user + pk: 3 + fields: + username: "alanallgroup" + first_name: "Alan" + last_name: "Allgroup" + is_active: false + groups: + - 1 + - 2 + - 3 + +- model: auth.user + pk: 4 + fields: + username: "sam" + first_name: "Samuel" + last_name: "Salesperson" + groups: + - 3 + is_staff: true + is_superuser: true diff --git a/tasks.py b/tasks.py index 7a6e952b15..81c4ab5678 100644 --- a/tasks.py +++ b/tasks.py @@ -286,6 +286,8 @@ def export_records(c, filename='data.json'): # Dump data to temporary file manage(c, cmd, pty=True) + print("Running data post-processing step...") + # Post-process the file, to remove any "permissions" specified for a group with open(tmpfile, "r") as f_in: data = json.loads(f_in.read()) @@ -365,6 +367,9 @@ def import_fixtures(c): 'location', 'stock_tests', 'stock', + + # Users + 'users' ] command = 'loaddata ' + ' '.join(fixtures) From b6cfa50e127d3f74247ab148219ef6336e445cca Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 11:59:35 +1000 Subject: [PATCH 18/23] Allow data operations to run for 'test' --- InvenTree/InvenTree/ready.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index ae9ddacdfc..6facedd6ff 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -29,7 +29,6 @@ def canAppAccessDatabase(): 'collectstatic', 'makemessages', 'compilemessages', - 'test', ] for cmd in excluded_commands: From 2f2354afdc83ec1e89e96c6419b9992411c6e319 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 12:07:58 +1000 Subject: [PATCH 19/23] Pre-process json data before importing - Removes any unwanted "permissions" entries from data imported from an *older* version of InvenTree --- .gitignore | 1 + tasks.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a2d5ef04b8..54ad8f07b6 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ config.yaml # Default data file data.json *.json.tmp +*.tmp.json # Key file secret_key.txt diff --git a/tasks.py b/tasks.py index 81c4ab5678..0d2f820bea 100644 --- a/tasks.py +++ b/tasks.py @@ -288,7 +288,7 @@ def export_records(c, filename='data.json'): print("Running data post-processing step...") - # Post-process the file, to remove any "permissions" specified for a group + # Post-process the file, to remove any "permissions" specified for a user or group with open(tmpfile, "r") as f_in: data = json.loads(f_in.read()) @@ -307,6 +307,8 @@ def export_records(c, filename='data.json'): with open(filename, "w") as f_out: f_out.write(json.dumps(data, indent=2)) + print("Data export completed") + @task(help={'filename': 'Input filename'}) def import_records(c, filename='data.json'): @@ -324,10 +326,33 @@ def import_records(c, filename='data.json'): print(f"Importing database records from '{filename}'") - cmd = f"loaddata {filename} -i {content_excludes()}" + # Pre-process the data, to remove any "permissions" specified for a user or group + tmpfile = f"{filename}.tmp.json" + + with open(filename, "r") as f_in: + data = json.loads(f_in.read()) + + for entry in data: + if "model" in entry: + + # Clear out any permissions specified for a group + if entry["model"] == "auth.group": + entry["fields"]["permissions"] = [] + + # Clear out any permissions specified for a user + if entry["model"] == "auth.user": + entry["fields"]["user_permissions"] = [] + + # Write the processed data to the tmp file + with open(tmpfile, "w") as f_out: + f_out.write(json.dumps(data, indent=2)) + + cmd = f"loaddata {tmpfile} -i {content_excludes()}" manage(c, cmd, pty=True) + print("Data import completed") + @task def import_fixtures(c): """ From b08cd8da20c66ca5494cc435a745389afd3ab927 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 12:14:36 +1000 Subject: [PATCH 20/23] Remove dbbackup integration - Data dumping and restoring is now very complex! - We should use the invoke export-records function now, rather than relying on dbbackup / dbrestore - Documentation will be updated to match --- InvenTree/InvenTree/ready.py | 2 -- InvenTree/InvenTree/settings.py | 12 ------------ InvenTree/config_template.yaml | 8 +------- docker/Dockerfile | 1 - docker/start_dev_server.sh | 5 ----- docker/start_prod_server.sh | 5 ----- requirements.txt | 1 - tasks.py | 21 --------------------- 8 files changed, 1 insertion(+), 54 deletions(-) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index 6facedd6ff..f78c7ad493 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -18,8 +18,6 @@ def canAppAccessDatabase(): 'makemirations', 'migrate', 'check', - 'dbbackup', - 'mediabackup', 'dbrestore', 'mediarestore', 'shell', diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 596dbd29c8..8d9012b762 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -250,7 +250,6 @@ INSTALLED_APPS = [ # Third part add-ons 'django_filters', # Extended filter functionality - 'dbbackup', # Database backup / restore 'rest_framework', # DRF (Django Rest Framework) 'rest_framework.authtoken', # Token authentication for API 'corsheaders', # Cross-origin Resource Sharing for DRF @@ -586,17 +585,6 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3' # Use database transactions when importing / exporting data IMPORT_EXPORT_USE_TRANSACTIONS = True -BACKUP_DIR = get_setting( - 'INVENTREE_BACKUP_DIR', - CONFIG.get('backup_dir', tempfile.gettempdir()), -) - -# Settings for dbbsettings app -DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage' -DBBACKUP_STORAGE_OPTIONS = { - 'location': BACKUP_DIR, -} - # Internal IP addresses allowed to see the debug toolbar INTERNAL_IPS = [ '127.0.0.1', diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index 0a86d88827..de09e37e4f 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -138,12 +138,6 @@ static_root: '/home/inventree/static' # - git # - ssh -# Backup options -# Set the backup_dir parameter to store backup files in a specific location -# If unspecified, the local user's temp directory will be used -# Use environment variable INVENTREE_BACKUP_DIR -backup_dir: '/home/inventree/data/backup/' - # Permit custom authentication backends #authentication_backends: # - 'django.contrib.auth.backends.ModelBackend' @@ -159,4 +153,4 @@ backup_dir: '/home/inventree/data/backup/' # - 'django.contrib.auth.middleware.AuthenticationMiddleware' # - 'django.contrib.messages.middleware.MessageMiddleware' # - 'django.middleware.clickjacking.XFrameOptionsMiddleware' -# - 'InvenTree.middleware.AuthRequiredMiddleware' \ No newline at end of file +# - 'InvenTree.middleware.AuthRequiredMiddleware' diff --git a/docker/Dockerfile b/docker/Dockerfile index ab4dbba6b5..c95c2867df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,6 @@ ENV INVENTREE_MNG_DIR="${INVENTREE_SRC_DIR}/InvenTree" ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data" ENV INVENTREE_STATIC_ROOT="${INVENTREE_HOME}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media" -ENV INVENTREE_BACKUP_DIR="${INVENTREE_DATA_DIR}/backup" ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt" diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh index 481da3c31a..c22805b90b 100644 --- a/docker/start_dev_server.sh +++ b/docker/start_dev_server.sh @@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then mkdir $INVENTREE_MEDIA_ROOT fi -if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then - echo "Creating directory $INVENTREE_BACKUP_DIR" - mkdir $INVENTREE_BACKUP_DIR -fi - # Check if "config.yaml" has been copied into the correct location if test -f "$INVENTREE_CONFIG_FILE"; then echo "$INVENTREE_CONFIG_FILE exists - skipping" diff --git a/docker/start_prod_server.sh b/docker/start_prod_server.sh index 811e189d13..2767e844d6 100644 --- a/docker/start_prod_server.sh +++ b/docker/start_prod_server.sh @@ -11,11 +11,6 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then mkdir $INVENTREE_MEDIA_ROOT fi -if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then - echo "Creating directory $INVENTREE_BACKUP_DIR" - mkdir $INVENTREE_BACKUP_DIR -fi - # Check if "config.yaml" has been copied into the correct location if test -f "$INVENTREE_CONFIG_FILE"; then echo "$INVENTREE_CONFIG_FILE exists - skipping" diff --git a/requirements.txt b/requirements.txt index 1392531e29..3291574084 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ wheel>=0.34.2 # Wheel Django==3.2 # Django package pillow==8.1.1 # Image manipulation djangorestframework==3.12.4 # DRF framework -django-dbbackup==3.3.0 # Database backup / restore functionality django-cors-headers==3.2.0 # CORS headers extension for DRF django-filter==2.4.0 # Extended filtering options django-mptt==0.11.0 # Modified Preorder Tree Traversal diff --git a/tasks.py b/tasks.py index 0d2f820bea..c4eda5e5bf 100644 --- a/tasks.py +++ b/tasks.py @@ -401,27 +401,6 @@ def import_fixtures(c): manage(c, command, pty=True) -@task -def backup(c): - """ - Create a backup of database models and uploaded media files. - - Backup files will be written to the 'backup_dir' file specified in 'config.yaml' - """ - - manage(c, 'dbbackup') - manage(c, 'mediabackup') - -@task -def restore(c): - """ - Restores database models and media files. - - Backup files are read from the 'backup_dir' file specified in 'config.yaml' - """ - - manage(c, 'dbrestore') - manage(c, 'mediarestore') @task(help={'address': 'Server address:port (default=127.0.0.1:8000)'}) def server(c, address="127.0.0.1:8000"): From f8ec32c3f55502375ec1fed15f018195a1439a67 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 12:24:05 +1000 Subject: [PATCH 21/23] Remove unused include --- InvenTree/InvenTree/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 8d9012b762..4fd8efae1a 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -17,7 +17,6 @@ import random import string import shutil import sys -import tempfile from datetime import datetime import yaml From 5dbebf37b7c30c9d600876ac04e6381f6caf4c33 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 12:30:14 +1000 Subject: [PATCH 22/23] Duplicate import of the same data should *NOT* error out --- .github/workflows/coverage.yaml | 1 + .github/workflows/mysql.yaml | 1 + .github/workflows/postgresql.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index d903b737e4..ad5f7a841e 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -44,6 +44,7 @@ jobs: rm test_db.sqlite invoke migrate invoke import-records -f data.json + invoke import-records -f data.json - name: Test Translations run: invoke translate - name: Check Migration Files diff --git a/.github/workflows/mysql.yaml b/.github/workflows/mysql.yaml index e4fd7ed81d..087a866fbd 100644 --- a/.github/workflows/mysql.yaml +++ b/.github/workflows/mysql.yaml @@ -56,4 +56,5 @@ jobs: invoke import-fixtures invoke export-records -f data.json python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json invoke import-records -f data.json \ No newline at end of file diff --git a/.github/workflows/postgresql.yaml b/.github/workflows/postgresql.yaml index 3938457722..ae8f52b962 100644 --- a/.github/workflows/postgresql.yaml +++ b/.github/workflows/postgresql.yaml @@ -52,4 +52,5 @@ jobs: invoke import-fixtures invoke export-records -f data.json python3 ./InvenTree/manage.py flush --noinput + invoke import-records -f data.json invoke import-records -f data.json \ No newline at end of file From 80f11290c91681160a7e018b36722278f41b9681 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 25 Apr 2021 12:30:59 +1000 Subject: [PATCH 23/23] Remove old command --- InvenTree/InvenTree/ready.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index f78c7ad493..aa31fac947 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -18,7 +18,6 @@ def canAppAccessDatabase(): 'makemirations', 'migrate', 'check', - 'dbrestore', 'mediarestore', 'shell', 'createsuperuser',