Refactor label/report template copying (#6582)

* [BUG] Inventree fiddles with files directly rather than using Django Storage api
Fixes #2585

* PEP fix

* clean diff

* move template discovery into central location

* more moving file operations

* fix paths

* and another path fixing

* more fixes

* fix typing

* switch config back to local

* revert locale stats

* add s3 support

* storages

* more adaptions

* use s3 switch to set storage backend

* fix reqs

* cleanup default_storage

* init in storage_backend

* move to storage classes everywhere

* fix call

* remove more S3 references

* move storage init

* fix startup error

* alsways use url

* ignore FileExistsError

* move s3 required url in

* remove S3 for now

* use Djangos defaults

* fix old import

* remove default_storage calls

* make labels/reports more similar

* expand functions out

* refactor to use refs where possible

* refactor copy section to be similar

* unify db lookup

* move shared code to generic section

* move ready out

* docstrings

* move even more functions out

* move references inline of the classes

* clean up refs

* fix init

* fix ensure dir

* remove unneeded tries

* cleanup diff

* more cleanup

* fix tests

* use SUBDIR
This commit is contained in:
Matthias Mair 2024-02-27 23:25:01 +01:00 committed by GitHub
parent f6123cc261
commit 1199291835
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 283 additions and 357 deletions

View File

@ -10,6 +10,9 @@ import string
import warnings
from pathlib import Path
from django.core.files.base import ContentFile
from django.core.files.storage import Storage
logger = logging.getLogger('inventree')
CONFIG_DATA = None
CONFIG_LOOKUPS = {}
@ -69,11 +72,16 @@ def get_base_dir() -> Path:
return Path(__file__).parent.parent.resolve()
def ensure_dir(path: Path) -> None:
def ensure_dir(path: Path, storage=None) -> None:
"""Ensure that a directory exists.
If it does not exist, create it.
"""
if storage and isinstance(storage, Storage):
if not storage.exists(str(path)):
storage.save(str(path / '.empty'), ContentFile(''))
return
if not path.exists():
path.mkdir(parents=True, exist_ok=True)

View File

View File

@ -0,0 +1,140 @@
"""Shared templating code."""
import logging
import os
import warnings
from pathlib import Path
from django.conf import settings
from django.core.exceptions import AppRegistryNotReady
from django.core.files.storage import default_storage
from django.db.utils import IntegrityError, OperationalError, ProgrammingError
from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
import InvenTree.helpers
from InvenTree.config import ensure_dir
logger = logging.getLogger('inventree')
MEDIA_STORAGE_DIR = Path(settings.MEDIA_ROOT)
class TemplatingMixin:
"""Mixin that contains shared templating code."""
name: str = ''
db: str = ''
def __init__(self, *args, **kwargs):
"""Ensure that the required properties are set."""
super().__init__(*args, **kwargs)
if self.name == '':
raise NotImplementedError('ref must be set')
if self.db == '':
raise NotImplementedError('db must be set')
def create_defaults(self):
"""Function that creates all default templates for the app."""
raise NotImplementedError('create_defaults must be implemented')
def get_src_dir(self, ref_name):
"""Get the source directory for the default templates."""
raise NotImplementedError('get_src_dir must be implemented')
def get_new_obj_data(self, data, filename):
"""Get the data for a new template db object."""
raise NotImplementedError('get_new_obj_data must be implemented')
# Standardized code
def ready(self):
"""This function is called whenever the app is loaded."""
import InvenTree.ready
# skip loading if plugin registry is not loaded or we run in a background thread
if (
not InvenTree.ready.isPluginRegistryLoaded()
or not InvenTree.ready.isInMainThread()
):
return
if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
return # pragma: no cover
with maintenance_mode_on():
try:
self.create_defaults()
except (
AppRegistryNotReady,
IntegrityError,
OperationalError,
ProgrammingError,
):
# Database might not yet be ready
warnings.warn(
f'Database was not ready for creating {self.name}s', stacklevel=2
)
set_maintenance_mode(False)
def create_template_dir(self, model, data):
"""Create folder and database entries for the default templates, if they do not already exist."""
ref_name = model.getSubdir()
# Create root dir for templates
src_dir = self.get_src_dir(ref_name)
dst_dir = MEDIA_STORAGE_DIR.joinpath(self.name, 'inventree', ref_name)
ensure_dir(dst_dir, default_storage)
# Copy each template across (if required)
for entry in data:
self.create_template_file(model, src_dir, entry, ref_name)
def create_template_file(self, model, src_dir, data, ref_name):
"""Ensure a label template is in place."""
# Destination filename
filename = os.path.join(self.name, 'inventree', ref_name, data['file'])
src_file = src_dir.joinpath(data['file'])
dst_file = MEDIA_STORAGE_DIR.joinpath(filename)
do_copy = False
if not dst_file.exists():
logger.info("%s template '%s' is not present", self.name, filename)
do_copy = True
else:
# Check if the file contents are different
src_hash = InvenTree.helpers.hash_file(src_file)
dst_hash = InvenTree.helpers.hash_file(dst_file)
if src_hash != dst_hash:
logger.info("Hash differs for '%s'", filename)
do_copy = True
if do_copy:
logger.info("Copying %s template '%s'", self.name, dst_file)
# Ensure destination dir exists
dst_file.parent.mkdir(parents=True, exist_ok=True)
# Copy file
default_storage.save(filename, src_file.open('rb'))
# Check if a file matching the template already exists
try:
if model.objects.filter(**{self.db: filename}).exists():
return # pragma: no cover
except Exception:
logger.exception(
"Failed to query %s for '%s' - you should run 'invoke update' first!",
self.name,
filename,
)
logger.info("Creating entry for %s '%s'", model, data.get('name'))
try:
model.objects.create(**self.get_new_obj_data(data, filename))
except Exception:
logger.warning("Failed to create %s '%s'", self.name, data['name'])

View File

@ -1,69 +1,31 @@
"""label app specification."""
"""Config options for the label app."""
import hashlib
import logging
import os
import shutil
import warnings
from pathlib import Path
from django.apps import AppConfig
from django.conf import settings
from django.core.exceptions import AppRegistryNotReady
from django.db.utils import IntegrityError, OperationalError, ProgrammingError
from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
import InvenTree.helpers
import InvenTree.ready
logger = logging.getLogger('inventree')
from generic.templating.apps import TemplatingMixin
class LabelConfig(AppConfig):
"""App configuration class for the 'label' app."""
class LabelConfig(TemplatingMixin, AppConfig):
"""Configuration class for the "label" app."""
name = 'label'
db = 'label'
def ready(self):
"""This function is called whenever the label app is loaded."""
# skip loading if plugin registry is not loaded or we run in a background thread
if (
not InvenTree.ready.isPluginRegistryLoaded()
or not InvenTree.ready.isInMainThread()
):
return
if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
return # pragma: no cover
with maintenance_mode_on():
try:
self.create_labels() # pragma: no cover
except (
AppRegistryNotReady,
IntegrityError,
OperationalError,
ProgrammingError,
):
# Database might not yet be ready
warnings.warn(
'Database was not ready for creating labels', stacklevel=2
)
set_maintenance_mode(False)
def create_labels(self):
def create_defaults(self):
"""Create all default templates."""
# Test if models are ready
import label.models
try:
import label.models
except Exception: # pragma: no cover
# Database is not ready yet
return
assert bool(label.models.StockLocationLabel is not None)
# Create the categories
self.create_labels_category(
self.create_template_dir(
label.models.StockItemLabel,
'stockitem',
[
{
'file': 'qr.html',
@ -75,9 +37,8 @@ class LabelConfig(AppConfig):
],
)
self.create_labels_category(
self.create_template_dir(
label.models.StockLocationLabel,
'stocklocation',
[
{
'file': 'qr.html',
@ -96,9 +57,8 @@ class LabelConfig(AppConfig):
],
)
self.create_labels_category(
self.create_template_dir(
label.models.PartLabel,
'part',
[
{
'file': 'part_label.html',
@ -117,9 +77,8 @@ class LabelConfig(AppConfig):
],
)
self.create_labels_category(
self.create_template_dir(
label.models.BuildLineLabel,
'buildline',
[
{
'file': 'buildline_label.html',
@ -131,72 +90,18 @@ class LabelConfig(AppConfig):
],
)
def create_labels_category(self, model, ref_name, labels):
"""Create folder and database entries for the default templates, if they do not already exist."""
# Create root dir for templates
src_dir = Path(__file__).parent.joinpath('templates', 'label', ref_name)
def get_src_dir(self, ref_name):
"""Get the source directory."""
return Path(__file__).parent.joinpath('templates', self.name, ref_name)
dst_dir = settings.MEDIA_ROOT.joinpath('label', 'inventree', ref_name)
if not dst_dir.exists():
logger.info("Creating required directory: '%s'", dst_dir)
dst_dir.mkdir(parents=True, exist_ok=True)
# Create labels
for label in labels:
self.create_template_label(model, src_dir, ref_name, label)
def create_template_label(self, model, src_dir, ref_name, label):
"""Ensure a label template is in place."""
filename = os.path.join('label', 'inventree', ref_name, label['file'])
src_file = src_dir.joinpath(label['file'])
dst_file = settings.MEDIA_ROOT.joinpath(filename)
to_copy = False
if dst_file.exists():
# File already exists - let's see if it is the "same"
if InvenTree.helpers.hash_file(dst_file) != InvenTree.helpers.hash_file(
src_file
): # pragma: no cover
logger.info("Hash differs for '%s'", filename)
to_copy = True
else:
logger.info("Label template '%s' is not present", filename)
to_copy = True
if to_copy:
logger.info("Copying label template '%s'", dst_file)
# Ensure destination dir exists
dst_file.parent.mkdir(parents=True, exist_ok=True)
# Copy file
shutil.copyfile(src_file, dst_file)
# Check if a label matching the template already exists
try:
if model.objects.filter(label=filename).exists():
return # pragma: no cover
except Exception:
logger.exception(
"Failed to query label for '%s' - you should run 'invoke update' first!",
filename,
)
logger.info("Creating entry for %s '%s'", model, label['name'])
try:
model.objects.create(
name=label['name'],
description=label['description'],
label=filename,
filters='',
enabled=True,
width=label['width'],
height=label['height'],
)
except Exception:
logger.warning("Failed to create label '%s'", label['name'])
def get_new_obj_data(self, data, filename):
"""Get the data for a new template db object."""
return {
'name': data['name'],
'description': data['description'],
'label': filename,
'filters': '',
'enabled': True,
'width': data['width'],
'height': data['height'],
}

View File

@ -96,8 +96,13 @@ class LabelTemplate(InvenTree.models.InvenTreeMetadataModel):
abstract = True
@classmethod
def getSubdir(cls) -> str:
"""Return the subdirectory for this label."""
return cls.SUBDIR
# Each class of label files will be stored in a separate subdirectory
SUBDIR = 'label'
SUBDIR: str = 'label'
# Object we will be printing against (will be filled out later)
object_to_print = None

View File

@ -30,7 +30,7 @@ class LabelTest(InvenTreeAPITestCase):
def setUpTestData(cls):
"""Ensure that some label instances exist as part of init routine."""
super().setUpTestData()
apps.get_app_config('label').create_labels()
apps.get_app_config('label').create_defaults()
def test_default_labels(self):
"""Test that the default label templates are copied across."""

View File

@ -247,7 +247,7 @@ class TestLabelPrinterMachineType(TestMachineRegistryMixin, InvenTreeAPITestCase
plugin_ref = 'inventreelabelmachine'
# setup the label app
apps.get_app_config('label').create_labels() # type: ignore
apps.get_app_config('label').create_defaults() # type: ignore
plg_registry.reload_plugins()
config = cast(PluginConfig, plg_registry.get_plugin(plugin_ref).plugin_config()) # type: ignore
config.active = True

View File

@ -121,7 +121,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
def test_printing_process(self):
"""Test that a label can be printed."""
# Ensure the labels were created
apps.get_app_config('label').create_labels()
apps.get_app_config('label').create_defaults()
# Lookup references
part = Part.objects.first()
@ -183,7 +183,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
def test_printing_options(self):
"""Test printing options."""
# Ensure the labels were created
apps.get_app_config('label').create_labels()
apps.get_app_config('label').create_defaults()
# Lookup references
parts = Part.objects.all()[:2]
@ -224,7 +224,7 @@ class LabelMixinTests(InvenTreeAPITestCase):
plugin_ref = 'samplelabelprinter'
# Activate the label components
apps.get_app_config('label').create_labels()
apps.get_app_config('label').create_defaults()
self.do_activate_plugin()
def run_print_test(label, qs, url_name, url_single):

View File

@ -1,256 +1,124 @@
"""Config options for the 'report' app."""
"""Config options for the report app."""
import logging
import os
import shutil
import warnings
from pathlib import Path
from django.apps import AppConfig
from django.conf import settings
from django.core.exceptions import AppRegistryNotReady
from django.db.utils import IntegrityError, OperationalError, ProgrammingError
from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
import InvenTree.helpers
logger = logging.getLogger('inventree')
from generic.templating.apps import TemplatingMixin
class ReportConfig(AppConfig):
"""Configuration class for the 'report' app."""
class ReportConfig(TemplatingMixin, AppConfig):
"""Configuration class for the "report" app."""
name = 'report'
db = 'template'
def ready(self):
"""This function is called whenever the report app is loaded."""
import InvenTree.ready
# skip loading if plugin registry is not loaded or we run in a background thread
if (
not InvenTree.ready.isPluginRegistryLoaded()
or not InvenTree.ready.isInMainThread()
):
return
if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
return # pragma: no cover
"""This function is called whenever the app is loaded."""
# Configure logging for PDF generation (disable "info" messages)
logging.getLogger('fontTools').setLevel(logging.WARNING)
logging.getLogger('weasyprint').setLevel(logging.WARNING)
with maintenance_mode_on():
self.create_reports()
super().ready()
set_maintenance_mode(False)
def create_reports(self):
"""Create default report templates."""
def create_defaults(self):
"""Create all default templates."""
# Test if models are ready
try:
self.create_default_test_reports()
self.create_default_build_reports()
self.create_default_bill_of_materials_reports()
self.create_default_purchase_order_reports()
self.create_default_sales_order_reports()
self.create_default_return_order_reports()
self.create_default_stock_location_reports()
except (
AppRegistryNotReady,
IntegrityError,
OperationalError,
ProgrammingError,
):
# Database might not yet be ready
warnings.warn('Database was not ready for creating reports', stacklevel=2)
def create_default_reports(self, model, reports):
"""Copy default report files across to the media directory."""
# Source directory for report templates
src_dir = Path(__file__).parent.joinpath('templates', 'report')
# Destination directory
dst_dir = settings.MEDIA_ROOT.joinpath('report', 'inventree', model.getSubdir())
if not dst_dir.exists():
logger.info("Creating missing directory: '%s'", dst_dir)
dst_dir.mkdir(parents=True, exist_ok=True)
# Copy each report template across (if required)
for report in reports:
# Destination filename
filename = os.path.join(
'report', 'inventree', model.getSubdir(), report['file']
)
src_file = src_dir.joinpath(report['file'])
dst_file = settings.MEDIA_ROOT.joinpath(filename)
do_copy = False
if not dst_file.exists():
logger.info("Report template '%s' is not present", filename)
do_copy = True
else:
# Check if the file contents are different
src_hash = InvenTree.helpers.hash_file(src_file)
dst_hash = InvenTree.helpers.hash_file(dst_file)
if src_hash != dst_hash:
logger.info("Hash differs for '%s'", filename)
do_copy = True
if do_copy:
logger.info("Copying test report template '%s'", dst_file)
shutil.copyfile(src_file, dst_file)
try:
# Check if a report matching the template already exists
if model.objects.filter(template=filename).exists():
continue
logger.info("Creating new TestReport for '%s'", report.get('name'))
model.objects.create(
name=report['name'],
description=report['description'],
template=filename,
enabled=True,
)
except Exception:
pass
def create_default_test_reports(self):
"""Create database entries for the default TestReport templates, if they do not already exist."""
try:
from .models import TestReport
import report.models
except Exception: # pragma: no cover
# Database is not ready yet
return
assert bool(report.models.TestReport is not None)
# List of test reports to copy across
reports = [
{
'file': 'inventree_test_report.html',
'name': 'InvenTree Test Report',
'description': 'Stock item test report',
}
]
# Create the categories
self.create_template_dir(
report.models.TestReport,
[
{
'file': 'inventree_test_report.html',
'name': 'InvenTree Test Report',
'description': 'Stock item test report',
}
],
)
self.create_default_reports(TestReport, reports)
self.create_template_dir(
report.models.BuildReport,
[
{
'file': 'inventree_build_order.html',
'name': 'InvenTree Build Order',
'description': 'Build Order job sheet',
}
],
)
def create_default_bill_of_materials_reports(self):
"""Create database entries for the default Bill of Material templates (if they do not already exist)."""
try:
from .models import BillOfMaterialsReport
except Exception: # pragma: no cover
# Database is not ready yet
return
self.create_template_dir(
report.models.BillOfMaterialsReport,
[
{
'file': 'inventree_bill_of_materials_report.html',
'name': 'Bill of Materials',
'description': 'Bill of Materials report',
}
],
)
# List of Build reports to copy across
reports = [
{
'file': 'inventree_bill_of_materials_report.html',
'name': 'Bill of Materials',
'description': 'Bill of Materials report',
}
]
self.create_template_dir(
report.models.PurchaseOrderReport,
[
{
'file': 'inventree_po_report.html',
'name': 'InvenTree Purchase Order',
'description': 'Purchase Order example report',
}
],
)
self.create_default_reports(BillOfMaterialsReport, reports)
self.create_template_dir(
report.models.SalesOrderReport,
[
{
'file': 'inventree_so_report.html',
'name': 'InvenTree Sales Order',
'description': 'Sales Order example report',
}
],
)
def create_default_build_reports(self):
"""Create database entries for the default BuildReport templates (if they do not already exist)."""
try:
from .models import BuildReport
except Exception: # pragma: no cover
# Database is not ready yet
return
self.create_template_dir(
report.models.ReturnOrderReport,
[
{
'file': 'inventree_return_order_report.html',
'name': 'InvenTree Return Order',
'description': 'Return Order example report',
}
],
)
# List of Build reports to copy across
reports = [
{
'file': 'inventree_build_order.html',
'name': 'InvenTree Build Order',
'description': 'Build Order job sheet',
}
]
self.create_template_dir(
report.models.StockLocationReport,
[
{
'file': 'inventree_slr_report.html',
'name': 'InvenTree Stock Location',
'description': 'Stock Location example report',
}
],
)
self.create_default_reports(BuildReport, reports)
def get_src_dir(self, ref_name):
"""Get the source directory."""
return Path(__file__).parent.joinpath('templates', self.name)
def create_default_purchase_order_reports(self):
"""Create database entries for the default SalesOrderReport templates (if they do not already exist)."""
try:
from .models import PurchaseOrderReport
except Exception: # pragma: no cover
# Database is not ready yet
return
# List of Build reports to copy across
reports = [
{
'file': 'inventree_po_report.html',
'name': 'InvenTree Purchase Order',
'description': 'Purchase Order example report',
}
]
self.create_default_reports(PurchaseOrderReport, reports)
def create_default_sales_order_reports(self):
"""Create database entries for the default Sales Order report templates (if they do not already exist)."""
try:
from .models import SalesOrderReport
except Exception: # pragma: no cover
# Database is not ready yet
return
# List of Build reports to copy across
reports = [
{
'file': 'inventree_so_report.html',
'name': 'InvenTree Sales Order',
'description': 'Sales Order example report',
}
]
self.create_default_reports(SalesOrderReport, reports)
def create_default_return_order_reports(self):
"""Create database entries for the default ReturnOrderReport templates."""
try:
from report.models import ReturnOrderReport
except Exception: # pragma: no cover
# Database not yet ready
return
# List of templates to copy across
reports = [
{
'file': 'inventree_return_order_report.html',
'name': 'InvenTree Return Order',
'description': 'Return Order example report',
}
]
self.create_default_reports(ReturnOrderReport, reports)
def create_default_stock_location_reports(self):
"""Create database entries for the default StockLocationReport templates."""
try:
from report.models import StockLocationReport
except Exception: # pragma: no cover
# Database not yet ready
return
# List of templates to copy across
reports = [
{
'file': 'inventree_slr_report.html',
'name': 'InvenTree Stock Location',
'description': 'Stock Location example report',
}
]
self.create_default_reports(StockLocationReport, reports)
def get_new_obj_data(self, data, filename):
"""Get the data for a new template db object."""
return {
'name': data['name'],
'description': data['description'],
'template': filename,
'enabled': True,
}