Merge pull request #1122 from SchrodingersGat/import-export-fix

Fixes for import / export of data

.travis.yml

@@ -30,11 +30,24 @@ before_install:
 script:
     - cd InvenTree && python3 manage.py makemigrations && cd ..
     - python3 ci/check_migration_files.py
+    # Run unit testing / code coverage tests
     - invoke coverage
+    # Run unit test for SQL database backend
     - cd InvenTree && python3 manage.py test --settings=InvenTree.ci_mysql && cd ..
+    # Run unit test for PostgreSQL database backend
     - cd InvenTree && python3 manage.py test --settings=InvenTree.ci_postgresql && cd ..
     - invoke translate
     - invoke style
+    # Create an empty database and fill it with test data
+    - rm inventree_default_db.sqlite3
+    - invoke migrate
+    - invoke import-fixtures
+    # Export database records
+    - invoke export-records -f data.json
+    # Create a new empty database and import the saved data
+    - rm inventree_default_db.sqlite3
+    - invoke migrate
+    - invoke import-records -f data.json
 
 after_success:
     - coveralls

InvenTree/common/apps.py

@@ -1,92 +1,10 @@
+# -*- coding: utf-8 -*-
+
 from django.apps import AppConfig
-from django.db.utils import OperationalError, ProgrammingError, IntegrityError
 
 
 class CommonConfig(AppConfig):
     name = 'common'
 
     def ready(self):
-
-        """ Will be called when the Common app is first loaded """
-        self.add_instance_name()
-        self.add_default_settings()
-
-    def add_instance_name(self):
-        """
-        Check if an InstanceName has been defined for this database.
-        If not, create a random one!
-        """
-
-        # See note above
-        from .models import InvenTreeSetting
-
-        """
-        Note: The "old" instance name was stored under the key 'InstanceName',
-        but has now been renamed to 'INVENTREE_INSTANCE'.
-        """
-
-        try:
-
-            # Quick exit if a value already exists for 'inventree_instance'
-            if InvenTreeSetting.objects.filter(key='INVENTREE_INSTANCE').exists():
-                return
-
-            # Default instance name
-            instance_name = InvenTreeSetting.get_default_value('INVENTREE_INSTANCE')
-
-            # Use the old name if it exists
-            if InvenTreeSetting.objects.filter(key='InstanceName').exists():
-                instance = InvenTreeSetting.objects.get(key='InstanceName')
-                instance_name = instance.value
-
-                # Delete the legacy key
-                instance.delete()
-
-            # Create new value
-            InvenTreeSetting.objects.create(
-                key='INVENTREE_INSTANCE',
-                value=instance_name
-            )
-
-        except (OperationalError, ProgrammingError, IntegrityError):
-            # Migrations have not yet been applied - table does not exist
-            pass
-
-    def add_default_settings(self):
-        """
-        Create all required settings, if they do not exist.
-        """
-
-        from .models import InvenTreeSetting
-
-        for key in InvenTreeSetting.GLOBAL_SETTINGS.keys():
-            try:
-                settings = InvenTreeSetting.objects.filter(key__iexact=key)
-
-                if settings.count() == 0:
-                    value = InvenTreeSetting.get_default_value(key)
-
-                    print(f"Creating default setting for {key} -> '{value}'")
-
-                    InvenTreeSetting.objects.create(
-                        key=key,
-                        value=value
-                    )
-
-                    return
-
-                elif settings.count() > 1:
-                    # Prevent multiple shadow copies of the same setting!
-                    for setting in settings[1:]:
-                        setting.delete()
-
-                # Ensure that the key has the correct case
-                setting = settings[0]
-
-                if not setting.key == key:
-                    setting.key = key
-                    setting.save()
-
-            except (OperationalError, ProgrammingError, IntegrityError):
-                # Table might not yet exist
-                pass
+        pass

InvenTree/common/fixtures/currency.yaml

@@ -1,16 +0,0 @@
-# Test fixtures for Currency objects
-
-- model: common.currency
-  fields:
-    symbol: '$'
-    suffix: 'AUD'
-    description: 'Australian Dollars'
-    base: True
-
-- model: common.currency
-  fields:
-    symbol: '$'
-    suffix: 'USD'
-    description: 'US Dollars'
-    base: False
-    value: 1.4

InvenTree/common/fixtures/settings.yaml

@@ -0,0 +1,13 @@
+# Sample settings objects
+
+- model: common.InvenTreeSetting
+  pk: 1
+  fields:
+    key: INVENTREE_INSTANCE
+    value: "My very first InvenTree Instance"
+
+- model: common.InvenTreeSetting
+  pk: 2
+  fields:
+    key: INVENTREE_COMPANY_NAME
+    value: "ACME Pty Ltd"

InvenTree/common/tests.py

@@ -4,20 +4,7 @@ from __future__ import unicode_literals
 from django.test import TestCase
 from django.contrib.auth import get_user_model
 
-from .models import Currency, InvenTreeSetting
-
-
-class CurrencyTest(TestCase):
-    """ Tests for Currency model """
-
-    fixtures = [
-        'currency',
-    ]
-
-    def test_currency(self):
-        # Simple test for now (improve this later!)
-
-        self.assertEqual(Currency.objects.count(), 2)
+from .models import InvenTreeSetting
 
 
 class SettingsTest(TestCase):
@@ -25,6 +12,10 @@ class SettingsTest(TestCase):
     Tests for the 'settings' model
     """
 
+    fixtures = [
+        'settings',
+    ]
+
     def setUp(self):
 
         User = get_user_model()
@@ -35,6 +26,20 @@ class SettingsTest(TestCase):
 
         self.client.login(username='username', password='password')
 
+    def test_settings_objects(self):
+
+        # There should be two settings objects in the database
+        settings = InvenTreeSetting.objects.all()
+
+        self.assertEqual(settings.count(), 2)
+
+        instance_name = InvenTreeSetting.objects.get(pk=1)
+        self.assertEqual(instance_name.key, 'INVENTREE_INSTANCE')
+        self.assertEqual(instance_name.value, 'My very first InvenTree Instance')
+
+        # Check object lookup (case insensitive)
+        self.assertEqual(InvenTreeSetting.get_setting_object('iNvEnTrEE_inSTanCE').pk, 1)
+
     def test_required_values(self):
         """
         - Ensure that every global setting has a name.

tasks.py

@@ -6,6 +6,7 @@ from shutil import copyfile
 import random
 import string
 import os
+import sys
 
 def apps():
     """
@@ -238,6 +239,96 @@ def postgresql(c):
     c.run('sudo apt-get install postgresql postgresql-contrib libpq-dev')
     c.run('pip3 install psycopg2')
 
+@task(help={'filename': "Output filename (default = 'data.json')"})
+def export_records(c, filename='data.json'):
+    """
+    Export all database records to a file
+    """
+
+    # Get an absolute path to the file
+    if not os.path.isabs(filename):
+        filename = os.path.join(localDir(), filename)
+        filename = os.path.abspath(filename) 
+
+    print(f"Exporting database records to file '{filename}'")
+
+    if os.path.exists(filename):
+        response = input("Warning: file already exists. Do you want to overwrite? [y/N]: ")
+        response = str(response).strip().lower()
+
+        if response not in ['y', 'yes']:
+            print("Cancelled export operation")
+            sys.exit(1)
+
+    cmd = f'dumpdata --exclude contenttypes --exclude auth.permission --indent 2 --output {filename}'
+
+    manage(c, cmd, pty=True)
+
+@task(help={'filename': 'Input filename'})
+def import_records(c, filename='data.json'):
+    """
+    Import database records from a file
+    """
+
+    # Get an absolute path to the supplied filename
+    if not os.path.isabs(filename):
+        filename = os.path.join(localDir(), filename)
+
+    if not os.path.exists(filename):
+        print(f"Error: File '{filename}' does not exist")
+        sys.exit(1)
+
+    print(f"Importing database records from '{filename}'")
+
+    cmd = f'loaddata {filename}'
+
+    manage(c, cmd, pty=True)
+
+@task
+def import_fixtures(c):
+    """
+    Import fixture data into the database.
+
+    This command imports all existing test fixture data into the database.
+
+    Warning:
+        - Intended for testing / development only!
+        - Running this command may overwrite existing database data!!
+        - Don't say you were not warned...
+    """
+
+    fixtures = [
+        # Build model
+        'build',
+        
+        # Common models
+        'settings',
+
+        # Company model
+        'company',
+        'price_breaks',
+        'supplier_part',
+
+        # Order model
+        'order',
+
+        # Part model
+        'bom',
+        'category',
+        'params',
+        'part',
+        'test_templates',
+
+        # Stock model
+        'location',
+        'stock_tests',
+        'stock',
+    ]
+
+    command = 'loaddata ' + ' '.join(fixtures)
+
+    manage(c, command, pty=True)
+
 @task
 def backup(c):
     """