Refactor label/report template copying (#6582)

* [BUG] Inventree fiddles with files directly rather than using Django Storage api
Fixes #2585

* PEP fix

* clean diff

* move template discovery into central location

* more moving file operations

* fix paths

* and another path fixing

* more fixes

* fix typing

* switch config back to local

* revert locale stats

* add s3 support

* storages

* more adaptions

* use s3 switch to set storage backend

* fix reqs

* cleanup default_storage

* init in storage_backend

* move to storage classes everywhere

* fix call

* remove more S3 references

* move storage init

* fix startup error

* alsways use url

* ignore FileExistsError

* move s3 required url in

* remove S3 for now

* use Djangos defaults

* fix old import

* remove default_storage calls

* make labels/reports more similar

* expand functions out

* refactor to use refs where possible

* refactor copy section to be similar

* unify db lookup

* move shared code to generic section

* move ready out

* docstrings

* move even more functions out

* move references inline of the classes

* clean up refs

* fix init

* fix ensure dir

* remove unneeded tries

* cleanup diff

* more cleanup

* fix tests

* use SUBDIR

InvenTree/InvenTree/config.py

@@ -10,6 +10,9 @@ import string
 import warnings
 from pathlib import Path
 
+from django.core.files.base import ContentFile
+from django.core.files.storage import Storage
+
 logger = logging.getLogger('inventree')
 CONFIG_DATA = None
 CONFIG_LOOKUPS = {}
@@ -69,11 +72,16 @@ def get_base_dir() -> Path:
     return Path(__file__).parent.parent.resolve()
 
 
-def ensure_dir(path: Path) -> None:
+def ensure_dir(path: Path, storage=None) -> None:
     """Ensure that a directory exists.
 
     If it does not exist, create it.
     """
+    if storage and isinstance(storage, Storage):
+        if not storage.exists(str(path)):
+            storage.save(str(path / '.empty'), ContentFile(''))
+        return
+
     if not path.exists():
         path.mkdir(parents=True, exist_ok=True)
 

InvenTree/generic/templating/apps.py

@@ -0,0 +1,140 @@
+"""Shared templating code."""
+
+import logging
+import os
+import warnings
+from pathlib import Path
+
+from django.conf import settings
+from django.core.exceptions import AppRegistryNotReady
+from django.core.files.storage import default_storage
+from django.db.utils import IntegrityError, OperationalError, ProgrammingError
+
+from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+
+import InvenTree.helpers
+from InvenTree.config import ensure_dir
+
+logger = logging.getLogger('inventree')
+
+
+MEDIA_STORAGE_DIR = Path(settings.MEDIA_ROOT)
+
+
+class TemplatingMixin:
+    """Mixin that contains shared templating code."""
+
+    name: str = ''
+    db: str = ''
+
+    def __init__(self, *args, **kwargs):
+        """Ensure that the required properties are set."""
+        super().__init__(*args, **kwargs)
+        if self.name == '':
+            raise NotImplementedError('ref must be set')
+        if self.db == '':
+            raise NotImplementedError('db must be set')
+
+    def create_defaults(self):
+        """Function that creates all default templates for the app."""
+        raise NotImplementedError('create_defaults must be implemented')
+
+    def get_src_dir(self, ref_name):
+        """Get the source directory for the default templates."""
+        raise NotImplementedError('get_src_dir must be implemented')
+
+    def get_new_obj_data(self, data, filename):
+        """Get the data for a new template db object."""
+        raise NotImplementedError('get_new_obj_data must be implemented')
+
+    # Standardized code
+    def ready(self):
+        """This function is called whenever the app is loaded."""
+        import InvenTree.ready
+
+        # skip loading if plugin registry is not loaded or we run in a background thread
+        if (
+            not InvenTree.ready.isPluginRegistryLoaded()
+            or not InvenTree.ready.isInMainThread()
+        ):
+            return
+
+        if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
+            return  # pragma: no cover
+
+        with maintenance_mode_on():
+            try:
+                self.create_defaults()
+            except (
+                AppRegistryNotReady,
+                IntegrityError,
+                OperationalError,
+                ProgrammingError,
+            ):
+                # Database might not yet be ready
+                warnings.warn(
+                    f'Database was not ready for creating {self.name}s', stacklevel=2
+                )
+
+        set_maintenance_mode(False)
+
+    def create_template_dir(self, model, data):
+        """Create folder and database entries for the default templates, if they do not already exist."""
+        ref_name = model.getSubdir()
+
+        # Create root dir for templates
+        src_dir = self.get_src_dir(ref_name)
+        dst_dir = MEDIA_STORAGE_DIR.joinpath(self.name, 'inventree', ref_name)
+        ensure_dir(dst_dir, default_storage)
+
+        # Copy each template across (if required)
+        for entry in data:
+            self.create_template_file(model, src_dir, entry, ref_name)
+
+    def create_template_file(self, model, src_dir, data, ref_name):
+        """Ensure a label template is in place."""
+        # Destination filename
+        filename = os.path.join(self.name, 'inventree', ref_name, data['file'])
+
+        src_file = src_dir.joinpath(data['file'])
+        dst_file = MEDIA_STORAGE_DIR.joinpath(filename)
+
+        do_copy = False
+
+        if not dst_file.exists():
+            logger.info("%s template '%s' is not present", self.name, filename)
+            do_copy = True
+        else:
+            # Check if the file contents are different
+            src_hash = InvenTree.helpers.hash_file(src_file)
+            dst_hash = InvenTree.helpers.hash_file(dst_file)
+
+            if src_hash != dst_hash:
+                logger.info("Hash differs for '%s'", filename)
+                do_copy = True
+
+        if do_copy:
+            logger.info("Copying %s template '%s'", self.name, dst_file)
+            # Ensure destination dir exists
+            dst_file.parent.mkdir(parents=True, exist_ok=True)
+
+            # Copy file
+            default_storage.save(filename, src_file.open('rb'))
+
+        # Check if a file matching the template already exists
+        try:
+            if model.objects.filter(**{self.db: filename}).exists():
+                return  # pragma: no cover
+        except Exception:
+            logger.exception(
+                "Failed to query %s for '%s' - you should run 'invoke update' first!",
+                self.name,
+                filename,
+            )
+
+        logger.info("Creating entry for %s '%s'", model, data.get('name'))
+
+        try:
+            model.objects.create(**self.get_new_obj_data(data, filename))
+        except Exception:
+            logger.warning("Failed to create %s '%s'", self.name, data['name'])

InvenTree/label/apps.py

@@ -1,69 +1,31 @@
-"""label app specification."""
+"""Config options for the label app."""
 
-import hashlib
-import logging
-import os
-import shutil
-import warnings
 from pathlib import Path
 
 from django.apps import AppConfig
-from django.conf import settings
-from django.core.exceptions import AppRegistryNotReady
-from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 
-from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+from generic.templating.apps import TemplatingMixin
-
-import InvenTree.helpers
-import InvenTree.ready
-
-logger = logging.getLogger('inventree')
 
 
-class LabelConfig(AppConfig):
+class LabelConfig(TemplatingMixin, AppConfig):
-    """App configuration class for the 'label' app."""
+    """Configuration class for the "label" app."""
 
     name = 'label'
+    db = 'label'
 
-    def ready(self):
+    def create_defaults(self):
-        """This function is called whenever the label app is loaded."""
-        # skip loading if plugin registry is not loaded or we run in a background thread
-        if (
-            not InvenTree.ready.isPluginRegistryLoaded()
-            or not InvenTree.ready.isInMainThread()
-        ):
-            return
-
-        if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
-            return  # pragma: no cover
-
-        with maintenance_mode_on():
-            try:
-                self.create_labels()  # pragma: no cover
-            except (
-                AppRegistryNotReady,
-                IntegrityError,
-                OperationalError,
-                ProgrammingError,
-            ):
-                # Database might not yet be ready
-                warnings.warn(
-                    'Database was not ready for creating labels', stacklevel=2
-                )
-
-        set_maintenance_mode(False)
-
-    def create_labels(self):
         """Create all default templates."""
         # Test if models are ready
-        import label.models
+        try:
-
+            import label.models
+        except Exception:  # pragma: no cover
+            # Database is not ready yet
+            return
         assert bool(label.models.StockLocationLabel is not None)
 
         # Create the categories
-        self.create_labels_category(
+        self.create_template_dir(
             label.models.StockItemLabel,
-            'stockitem',
             [
                 {
                     'file': 'qr.html',
@@ -75,9 +37,8 @@ class LabelConfig(AppConfig):
             ],
         )
 
-        self.create_labels_category(
+        self.create_template_dir(
             label.models.StockLocationLabel,
-            'stocklocation',
             [
                 {
                     'file': 'qr.html',
@@ -96,9 +57,8 @@ class LabelConfig(AppConfig):
             ],
         )
 
-        self.create_labels_category(
+        self.create_template_dir(
             label.models.PartLabel,
-            'part',
             [
                 {
                     'file': 'part_label.html',
@@ -117,9 +77,8 @@ class LabelConfig(AppConfig):
             ],
         )
 
-        self.create_labels_category(
+        self.create_template_dir(
             label.models.BuildLineLabel,
-            'buildline',
             [
                 {
                     'file': 'buildline_label.html',
@@ -131,72 +90,18 @@ class LabelConfig(AppConfig):
             ],
         )
 
-    def create_labels_category(self, model, ref_name, labels):
+    def get_src_dir(self, ref_name):
-        """Create folder and database entries for the default templates, if they do not already exist."""
+        """Get the source directory."""
-        # Create root dir for templates
+        return Path(__file__).parent.joinpath('templates', self.name, ref_name)
-        src_dir = Path(__file__).parent.joinpath('templates', 'label', ref_name)
 
-        dst_dir = settings.MEDIA_ROOT.joinpath('label', 'inventree', ref_name)
+    def get_new_obj_data(self, data, filename):
-
+        """Get the data for a new template db object."""
-        if not dst_dir.exists():
+        return {
-            logger.info("Creating required directory: '%s'", dst_dir)
+            'name': data['name'],
-            dst_dir.mkdir(parents=True, exist_ok=True)
+            'description': data['description'],
-
+            'label': filename,
-        # Create labels
+            'filters': '',
-        for label in labels:
+            'enabled': True,
-            self.create_template_label(model, src_dir, ref_name, label)
+            'width': data['width'],
-
+            'height': data['height'],
-    def create_template_label(self, model, src_dir, ref_name, label):
+        }
-        """Ensure a label template is in place."""
-        filename = os.path.join('label', 'inventree', ref_name, label['file'])
-
-        src_file = src_dir.joinpath(label['file'])
-        dst_file = settings.MEDIA_ROOT.joinpath(filename)
-
-        to_copy = False
-
-        if dst_file.exists():
-            # File already exists - let's see if it is the "same"
-
-            if InvenTree.helpers.hash_file(dst_file) != InvenTree.helpers.hash_file(
-                src_file
-            ):  # pragma: no cover
-                logger.info("Hash differs for '%s'", filename)
-                to_copy = True
-
-        else:
-            logger.info("Label template '%s' is not present", filename)
-            to_copy = True
-
-        if to_copy:
-            logger.info("Copying label template '%s'", dst_file)
-            # Ensure destination dir exists
-            dst_file.parent.mkdir(parents=True, exist_ok=True)
-
-            # Copy file
-            shutil.copyfile(src_file, dst_file)
-
-        # Check if a label matching the template already exists
-        try:
-            if model.objects.filter(label=filename).exists():
-                return  # pragma: no cover
-        except Exception:
-            logger.exception(
-                "Failed to query label for '%s' - you should run 'invoke update' first!",
-                filename,
-            )
-
-        logger.info("Creating entry for %s '%s'", model, label['name'])
-
-        try:
-            model.objects.create(
-                name=label['name'],
-                description=label['description'],
-                label=filename,
-                filters='',
-                enabled=True,
-                width=label['width'],
-                height=label['height'],
-            )
-        except Exception:
-            logger.warning("Failed to create label '%s'", label['name'])

InvenTree/label/models.py

@@ -96,8 +96,13 @@ class LabelTemplate(InvenTree.models.InvenTreeMetadataModel):
 
         abstract = True
 
+    @classmethod
+    def getSubdir(cls) -> str:
+        """Return the subdirectory for this label."""
+        return cls.SUBDIR
+
     # Each class of label files will be stored in a separate subdirectory
-    SUBDIR = 'label'
+    SUBDIR: str = 'label'
 
     # Object we will be printing against (will be filled out later)
     object_to_print = None

InvenTree/label/tests.py

@@ -30,7 +30,7 @@ class LabelTest(InvenTreeAPITestCase):
     def setUpTestData(cls):
         """Ensure that some label instances exist as part of init routine."""
         super().setUpTestData()
-        apps.get_app_config('label').create_labels()
+        apps.get_app_config('label').create_defaults()
 
     def test_default_labels(self):
         """Test that the default label templates are copied across."""

InvenTree/machine/tests.py

@@ -247,7 +247,7 @@ class TestLabelPrinterMachineType(TestMachineRegistryMixin, InvenTreeAPITestCase
         plugin_ref = 'inventreelabelmachine'
 
         # setup the label app
-        apps.get_app_config('label').create_labels()  # type: ignore
+        apps.get_app_config('label').create_defaults()  # type: ignore
         plg_registry.reload_plugins()
         config = cast(PluginConfig, plg_registry.get_plugin(plugin_ref).plugin_config())  # type: ignore
         config.active = True

InvenTree/plugin/base/label/test_label_mixin.py

@@ -121,7 +121,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
     def test_printing_process(self):
         """Test that a label can be printed."""
         # Ensure the labels were created
-        apps.get_app_config('label').create_labels()
+        apps.get_app_config('label').create_defaults()
 
         # Lookup references
         part = Part.objects.first()
@@ -183,7 +183,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
     def test_printing_options(self):
         """Test printing options."""
         # Ensure the labels were created
-        apps.get_app_config('label').create_labels()
+        apps.get_app_config('label').create_defaults()
 
         # Lookup references
         parts = Part.objects.all()[:2]
@@ -224,7 +224,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
         plugin_ref = 'samplelabelprinter'
 
         # Activate the label components
-        apps.get_app_config('label').create_labels()
+        apps.get_app_config('label').create_defaults()
         self.do_activate_plugin()
 
         def run_print_test(label, qs, url_name, url_single):

InvenTree/report/apps.py

@@ -1,256 +1,124 @@
-"""Config options for the 'report' app."""
+"""Config options for the report app."""
 
 import logging
-import os
-import shutil
-import warnings
 from pathlib import Path
 
 from django.apps import AppConfig
-from django.conf import settings
-from django.core.exceptions import AppRegistryNotReady
-from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 
-from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+from generic.templating.apps import TemplatingMixin
-
-import InvenTree.helpers
-
-logger = logging.getLogger('inventree')
 
 
-class ReportConfig(AppConfig):
+class ReportConfig(TemplatingMixin, AppConfig):
-    """Configuration class for the 'report' app."""
+    """Configuration class for the "report" app."""
 
     name = 'report'
+    db = 'template'
 
     def ready(self):
-        """This function is called whenever the report app is loaded."""
+        """This function is called whenever the app is loaded."""
-        import InvenTree.ready
-
-        # skip loading if plugin registry is not loaded or we run in a background thread
-        if (
-            not InvenTree.ready.isPluginRegistryLoaded()
-            or not InvenTree.ready.isInMainThread()
-        ):
-            return
-
-        if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
-            return  # pragma: no cover
-
         # Configure logging for PDF generation (disable "info" messages)
         logging.getLogger('fontTools').setLevel(logging.WARNING)
         logging.getLogger('weasyprint').setLevel(logging.WARNING)
 
-        with maintenance_mode_on():
+        super().ready()
-            self.create_reports()
 
-        set_maintenance_mode(False)
+    def create_defaults(self):
-
+        """Create all default templates."""
-    def create_reports(self):
+        # Test if models are ready
-        """Create default report templates."""
         try:
-            self.create_default_test_reports()
+            import report.models
-            self.create_default_build_reports()
-            self.create_default_bill_of_materials_reports()
-            self.create_default_purchase_order_reports()
-            self.create_default_sales_order_reports()
-            self.create_default_return_order_reports()
-            self.create_default_stock_location_reports()
-        except (
-            AppRegistryNotReady,
-            IntegrityError,
-            OperationalError,
-            ProgrammingError,
-        ):
-            # Database might not yet be ready
-            warnings.warn('Database was not ready for creating reports', stacklevel=2)
-
-    def create_default_reports(self, model, reports):
-        """Copy default report files across to the media directory."""
-        # Source directory for report templates
-        src_dir = Path(__file__).parent.joinpath('templates', 'report')
-
-        # Destination directory
-        dst_dir = settings.MEDIA_ROOT.joinpath('report', 'inventree', model.getSubdir())
-
-        if not dst_dir.exists():
-            logger.info("Creating missing directory: '%s'", dst_dir)
-            dst_dir.mkdir(parents=True, exist_ok=True)
-
-        # Copy each report template across (if required)
-        for report in reports:
-            # Destination filename
-            filename = os.path.join(
-                'report', 'inventree', model.getSubdir(), report['file']
-            )
-
-            src_file = src_dir.joinpath(report['file'])
-            dst_file = settings.MEDIA_ROOT.joinpath(filename)
-
-            do_copy = False
-
-            if not dst_file.exists():
-                logger.info("Report template '%s' is not present", filename)
-                do_copy = True
-            else:
-                # Check if the file contents are different
-                src_hash = InvenTree.helpers.hash_file(src_file)
-                dst_hash = InvenTree.helpers.hash_file(dst_file)
-
-                if src_hash != dst_hash:
-                    logger.info("Hash differs for '%s'", filename)
-                    do_copy = True
-
-            if do_copy:
-                logger.info("Copying test report template '%s'", dst_file)
-                shutil.copyfile(src_file, dst_file)
-
-            try:
-                # Check if a report matching the template already exists
-                if model.objects.filter(template=filename).exists():
-                    continue
-
-                logger.info("Creating new TestReport for '%s'", report.get('name'))
-
-                model.objects.create(
-                    name=report['name'],
-                    description=report['description'],
-                    template=filename,
-                    enabled=True,
-                )
-
-            except Exception:
-                pass
-
-    def create_default_test_reports(self):
-        """Create database entries for the default TestReport templates, if they do not already exist."""
-        try:
-            from .models import TestReport
         except Exception:  # pragma: no cover
             # Database is not ready yet
             return
+        assert bool(report.models.TestReport is not None)
 
-        # List of test reports to copy across
+        # Create the categories
-        reports = [
+        self.create_template_dir(
-            {
+            report.models.TestReport,
-                'file': 'inventree_test_report.html',
+            [
-                'name': 'InvenTree Test Report',
+                {
-                'description': 'Stock item test report',
+                    'file': 'inventree_test_report.html',
-            }
+                    'name': 'InvenTree Test Report',
-        ]
+                    'description': 'Stock item test report',
+                }
+            ],
+        )
 
-        self.create_default_reports(TestReport, reports)
+        self.create_template_dir(
+            report.models.BuildReport,
+            [
+                {
+                    'file': 'inventree_build_order.html',
+                    'name': 'InvenTree Build Order',
+                    'description': 'Build Order job sheet',
+                }
+            ],
+        )
 
-    def create_default_bill_of_materials_reports(self):
+        self.create_template_dir(
-        """Create database entries for the default Bill of Material templates (if they do not already exist)."""
+            report.models.BillOfMaterialsReport,
-        try:
+            [
-            from .models import BillOfMaterialsReport
+                {
-        except Exception:  # pragma: no cover
+                    'file': 'inventree_bill_of_materials_report.html',
-            # Database is not ready yet
+                    'name': 'Bill of Materials',
-            return
+                    'description': 'Bill of Materials report',
+                }
+            ],
+        )
 
-        # List of Build reports to copy across
+        self.create_template_dir(
-        reports = [
+            report.models.PurchaseOrderReport,
-            {
+            [
-                'file': 'inventree_bill_of_materials_report.html',
+                {
-                'name': 'Bill of Materials',
+                    'file': 'inventree_po_report.html',
-                'description': 'Bill of Materials report',
+                    'name': 'InvenTree Purchase Order',
-            }
+                    'description': 'Purchase Order example report',
-        ]
+                }
+            ],
+        )
 
-        self.create_default_reports(BillOfMaterialsReport, reports)
+        self.create_template_dir(
+            report.models.SalesOrderReport,
+            [
+                {
+                    'file': 'inventree_so_report.html',
+                    'name': 'InvenTree Sales Order',
+                    'description': 'Sales Order example report',
+                }
+            ],
+        )
 
-    def create_default_build_reports(self):
+        self.create_template_dir(
-        """Create database entries for the default BuildReport templates (if they do not already exist)."""
+            report.models.ReturnOrderReport,
-        try:
+            [
-            from .models import BuildReport
+                {
-        except Exception:  # pragma: no cover
+                    'file': 'inventree_return_order_report.html',
-            # Database is not ready yet
+                    'name': 'InvenTree Return Order',
-            return
+                    'description': 'Return Order example report',
+                }
+            ],
+        )
 
-        # List of Build reports to copy across
+        self.create_template_dir(
-        reports = [
+            report.models.StockLocationReport,
-            {
+            [
-                'file': 'inventree_build_order.html',
+                {
-                'name': 'InvenTree Build Order',
+                    'file': 'inventree_slr_report.html',
-                'description': 'Build Order job sheet',
+                    'name': 'InvenTree Stock Location',
-            }
+                    'description': 'Stock Location example report',
-        ]
+                }
+            ],
+        )
 
-        self.create_default_reports(BuildReport, reports)
+    def get_src_dir(self, ref_name):
+        """Get the source directory."""
+        return Path(__file__).parent.joinpath('templates', self.name)
 
-    def create_default_purchase_order_reports(self):
+    def get_new_obj_data(self, data, filename):
-        """Create database entries for the default SalesOrderReport templates (if they do not already exist)."""
+        """Get the data for a new template db object."""
-        try:
+        return {
-            from .models import PurchaseOrderReport
+            'name': data['name'],
-        except Exception:  # pragma: no cover
+            'description': data['description'],
-            # Database is not ready yet
+            'template': filename,
-            return
+            'enabled': True,
-
+        }
-        # List of Build reports to copy across
-        reports = [
-            {
-                'file': 'inventree_po_report.html',
-                'name': 'InvenTree Purchase Order',
-                'description': 'Purchase Order example report',
-            }
-        ]
-
-        self.create_default_reports(PurchaseOrderReport, reports)
-
-    def create_default_sales_order_reports(self):
-        """Create database entries for the default Sales Order report templates (if they do not already exist)."""
-        try:
-            from .models import SalesOrderReport
-        except Exception:  # pragma: no cover
-            # Database is not ready yet
-            return
-
-        # List of Build reports to copy across
-        reports = [
-            {
-                'file': 'inventree_so_report.html',
-                'name': 'InvenTree Sales Order',
-                'description': 'Sales Order example report',
-            }
-        ]
-
-        self.create_default_reports(SalesOrderReport, reports)
-
-    def create_default_return_order_reports(self):
-        """Create database entries for the default ReturnOrderReport templates."""
-        try:
-            from report.models import ReturnOrderReport
-        except Exception:  # pragma: no cover
-            # Database not yet ready
-            return
-
-        # List of templates to copy across
-        reports = [
-            {
-                'file': 'inventree_return_order_report.html',
-                'name': 'InvenTree Return Order',
-                'description': 'Return Order example report',
-            }
-        ]
-
-        self.create_default_reports(ReturnOrderReport, reports)
-
-    def create_default_stock_location_reports(self):
-        """Create database entries for the default StockLocationReport templates."""
-        try:
-            from report.models import StockLocationReport
-        except Exception:  # pragma: no cover
-            # Database not yet ready
-            return
-
-        # List of templates to copy across
-        reports = [
-            {
-                'file': 'inventree_slr_report.html',
-                'name': 'InvenTree Stock Location',
-                'description': 'Stock Location example report',
-            }
-        ]
-
-        self.create_default_reports(StockLocationReport, reports)