From f798537c7379822d6ee01474f3c7b168ca3a4123 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 09:52:59 +1100 Subject: [PATCH 01/42] Reverse migration company.0024 --- .../migrations/0024_unique_name_email_constraint.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/InvenTree/company/migrations/0024_unique_name_email_constraint.py b/InvenTree/company/migrations/0024_unique_name_email_constraint.py index 3a8781f98d..e6d1fd93dd 100644 --- a/InvenTree/company/migrations/0024_unique_name_email_constraint.py +++ b/InvenTree/company/migrations/0024_unique_name_email_constraint.py @@ -1,6 +1,14 @@ from django.db import migrations, models +def reverse_empty_email(apps, schema_editor): + Company = apps.get_model('company', 'Company') + for company in Company.objects.all(): + if company.email == None: + company.email = '' + company.save() + + def make_empty_email_field_null(apps, schema_editor): Company = apps.get_model('company', 'Company') for company in Company.objects.all(): @@ -23,7 +31,7 @@ class Migration(migrations.Migration): field=models.EmailField(blank=True, help_text='Contact email address', max_length=254, null=True, unique=False, verbose_name='Email'), ), # Convert empty email string to NULL - migrations.RunPython(make_empty_email_field_null), + migrations.RunPython(make_empty_email_field_null, reverse_code=reverse_empty_email), # Remove unique constraint on name field migrations.AlterField( model_name='company', From bc43d14ebfd5c455453aca55bc0183bc09c6dfda Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 11:28:43 +1100 Subject: [PATCH 02/42] Change model functions to raw SQL --- .../migrations/0019_auto_20200413_0642.py | 91 ++++++++++++------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index c3c2f58ea0..ced7ed04c3 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -112,50 +112,67 @@ def associate_manufacturers(apps, schema_editor): # Map company names to company objects companies = {} - for company in Company.objects.all(): - companies[company.name] = company + # Iterate through each company object + cursor = connection.cursor() + response = cursor.execute("select id, name from company_company;") + results = cursor.fetchall() - def link_part(part, name): + for index, row in enumerate(results): + pk, name = row + + companies[name] = pk + + def link_part(part_id, name): """ Attempt to link Part to an existing Company """ # Matches a company name directly if name in companies.keys(): - print(" - Part[{pk}]: '{n}' maps to existing manufacturer".format(pk=part.pk, n=name)) - part.manufacturer = companies[name] - part.save() + print(" - Part[{pk}]: '{n}' maps to existing manufacturer".format(pk=part_id, n=name)) + + manufacturer_id = companies[name] + + query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" + result = cursor.execute(query) + return True # Have we already mapped this if name in links.keys(): - print(" - Part[{pk}]: Mapped '{n}' - '{c}'".format(pk=part.pk, n=name, c=links[name].name)) - part.manufacturer = links[name] - part.save() + print(" - Part[{pk}]: Mapped '{n}' - '{c}'".format(pk=part_id, n=name, c=links[name].name)) + + query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" + result = query.execute() return True # Mapping not possible return False - def create_manufacturer(part, input_name, company_name): + def create_manufacturer(part_id, input_name, company_name): """ Create a new manufacturer """ - company = Company(name=company_name, description=company_name, is_manufacturer=True) + # Manually create a new database row + # Note: Have to fill out all empty string values! + new_manufacturer_query = f"insert into company_company ('name', 'description', 'is_customer', 'is_supplier', 'is_manufacturer', 'address', 'website', 'phone', 'email', 'contact', 'link', 'notes') values ('{company_name}', '{company_name}', false, false, true, '', '', '', '', '', '', '');" - company.is_manufacturer = True - - # Save the company BEFORE we associate the part, otherwise the PK does not exist - company.save() + cursor = connection.cursor() + + cursor.execute(new_manufacturer_query) + + # Extract the company back from the database + response = cursor.execute(f"select id from company_company where name='{company_name}';") + row = response.fetchone() + manufacturer_id = int(row[0]) # Map both names to the same company - links[input_name] = company - links[company_name] = company + links[input_name] = manufacturer_id + links[company_name] = manufacturer_id - companies[company_name] = company + companies[company_name] = manufacturer_id - print(" - Part[{pk}]: Created new manufacturer: '{name}'".format(pk=part.pk, name=company_name)) + print(" - Part[{pk}]: Created new manufacturer: '{name}'".format(pk=part_id, name=company_name)) - # Save the manufacturer reference link - part.manufacturer = company - part.save() + # Update SupplierPart object in the database + cursor.execute(f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};") def find_matches(text, threshold=65): """ @@ -178,17 +195,17 @@ def associate_manufacturers(apps, schema_editor): return [] - def map_part_to_manufacturer(part, idx, total): + def map_part_to_manufacturer(part_id, idx, total): - name = get_manufacturer_name(part.id) + name = get_manufacturer_name(part_id) # Skip empty names if not name or len(name) == 0: - print(" - Part[{pk}]: No manufacturer_name provided, skipping".format(pk=part.pk)) + print(" - Part[{pk}]: No manufacturer_name provided, skipping".format(pk=part_id)) return # Can be linked to an existing manufacturer - if link_part(part, name): + if link_part(part_id, name): return # Find a list of potential matches @@ -198,7 +215,7 @@ def associate_manufacturers(apps, schema_editor): # Present a list of options print("----------------------------------") - print("Checking part [{pk}] ({idx} of {total})".format(pk=part.pk, idx=idx+1, total=total)) + print("Checking part [{pk}] ({idx} of {total})".format(pk=part_id, idx=idx+1, total=total)) print("Manufacturer name: '{n}'".format(n=name)) print("----------------------------------") print("Select an option from the list below:") @@ -222,7 +239,7 @@ def associate_manufacturers(apps, schema_editor): # Option 0) is to create a new manufacturer with the current name if n == 0: - create_manufacturer(part, name, name) + create_manufacturer(part_id, name, name) return # Options 1) - n) select an existing manufacturer @@ -270,7 +287,7 @@ def associate_manufacturers(apps, schema_editor): # No match, create a new manufacturer else: - create_manufacturer(part, name, response) + create_manufacturer(part_id, name, response) return clear() @@ -292,16 +309,22 @@ def associate_manufacturers(apps, schema_editor): clear() - part_count = SupplierPart.objects.count() + # Extract all SupplierPart objects from the database + cursor = connection.cursor() + response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") + results = response.fetchall() + part_count = len(results) + # Create a unique set of manufacturer names - for idx, part in enumerate(SupplierPart.objects.all()): + for index, row in enumerate(results): + pk, MPN, SKU, manufacturer_id, manufacturer_name = row - if part.manufacturer is not None: - print(" - Part '{p}' already has a manufacturer associated (skipping)".format(p=part)) + if manufacturer_id is not None: + print(f" - SupplierPart '{pk}' already has a manufacturer associated (skipping)") continue - map_part_to_manufacturer(part, idx, part_count) + map_part_to_manufacturer(pk, index, part_count) print("Done!") From 793e5b820e4334837d9c068debc32ec838c86447 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 11:56:48 +1100 Subject: [PATCH 03/42] Remove all model references from migration file --- .../migrations/0019_auto_20200413_0642.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index ced7ed04c3..deda87a040 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -4,7 +4,6 @@ import os from rapidfuzz import fuzz from django.db import migrations, connection -from company.models import Company, SupplierPart from django.db.utils import OperationalError, ProgrammingError @@ -21,21 +20,25 @@ def reverse_association(apps, schema_editor): into the 'manufacturer_name' field. """ + cursor = connection.cursor() + + response = cursor.execute("select id, MPN from part_supplierpart;") + supplier_parts = response.fetchall() + # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! - if SupplierPart.objects.count() == 0: + if len(supplier_parts) == 0: return print("Reversing migration for manufacturer association") - for part in SupplierPart.objects.all(): + for (index, row) in enumerate(supplier_parts): + supplier_part_id, MPN = row - print("Checking part [{pk}]:".format(pk=part.pk)) - - cursor = connection.cursor() + print(f"Checking SupplierPart [{supplier_part_id}]:") # Grab the manufacturer ID from the part - response = cursor.execute('SELECT manufacturer_id FROM part_supplierpart WHERE id={ID};'.format(ID=part.id)) + response = cursor.execute(f"SELECT manufacturer_id FROM part_supplierpart WHERE id={supplier_part_id};") manufacturer_id = None @@ -54,7 +57,7 @@ def reverse_association(apps, schema_editor): print(" - Manufacturer ID: [{id}]".format(id=manufacturer_id)) # Now extract the "name" for the manufacturer - response = cursor.execute('SELECT name from company_company where id={ID};'.format(ID=manufacturer_id)) + response = cursor.execute(f"SELECT name from company_company where id={manufacturer_id};") row = response.fetchone() @@ -62,7 +65,7 @@ def reverse_association(apps, schema_editor): print(" - Manufacturer name: '{name}'".format(name=name)) - response = cursor.execute("UPDATE part_supplierpart SET manufacturer_name='{name}' WHERE id={ID};".format(name=name, ID=part.id)) + response = cursor.execute("UPDATE part_supplierpart SET manufacturer_name='{name}' WHERE id={ID};".format(name=name, ID=supplier_part_id)) def associate_manufacturers(apps, schema_editor): """ @@ -100,10 +103,14 @@ def associate_manufacturers(apps, schema_editor): return row[0] return '' + cursor = connection.cursor() + + response = cursor.execute("select id, MPN from part_supplierpart;") + supplier_parts = response.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! - if SupplierPart.objects.count() == 0: + if len(supplier_parts) == 0: return # Link a 'manufacturer_name' to a 'Company' @@ -113,7 +120,6 @@ def associate_manufacturers(apps, schema_editor): companies = {} # Iterate through each company object - cursor = connection.cursor() response = cursor.execute("select id, name from company_company;") results = cursor.fetchall() @@ -197,6 +203,8 @@ def associate_manufacturers(apps, schema_editor): def map_part_to_manufacturer(part_id, idx, total): + cursor = connection.cursor() + name = get_manufacturer_name(part_id) # Skip empty names @@ -249,21 +257,19 @@ def associate_manufacturers(apps, schema_editor): if n < len(matches): # Get the company which matches the selected options company_name = matches[n] - company = companies[company_name] + company_id = companies[company_name] # Ensure the company is designated as a manufacturer - company.is_manufacturer = True - company.save() + cursor.execute(f"update company_company set is_manufacturer=true where id={company_id};") # Link the company to the part - part.manufacturer = company - part.save() + cursor.execute(f"update part_supplierpart set manufacturer_id={company_id} where id={part_id};") # Link the name to the company - links[name] = company - links[company_name] = company + links[name] = company_id + links[company_name] = company_id - print(" - Part[{pk}]: Linked '{n}' to manufacturer '{m}'".format(pk=part.pk, n=name, m=company_name)) + print(" - Part[{pk}]: Linked '{n}' to manufacturer '{m}'".format(pk=part_id, n=name, m=company_name)) return else: @@ -321,7 +327,7 @@ def associate_manufacturers(apps, schema_editor): pk, MPN, SKU, manufacturer_id, manufacturer_name = row if manufacturer_id is not None: - print(f" - SupplierPart '{pk}' already has a manufacturer associated (skipping)") + print(f" - SupplierPart <{pk}> already has a manufacturer associated (skipping)") continue map_part_to_manufacturer(pk, index, part_count) From 0e246a7fdfb68335942deb8144d67c64677e2fc3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 13:02:28 +1100 Subject: [PATCH 04/42] Migration fix (response is different for postgresql) --- .../company/migrations/0019_auto_20200413_0642.py | 12 ++++++------ .../company/migrations/0026_auto_20201110_1011.py | 2 +- InvenTree/part/migrations/0056_auto_20201110_1125.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index deda87a040..a27f56ce3a 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -23,7 +23,7 @@ def reverse_association(apps, schema_editor): cursor = connection.cursor() response = cursor.execute("select id, MPN from part_supplierpart;") - supplier_parts = response.fetchall() + supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! @@ -42,7 +42,7 @@ def reverse_association(apps, schema_editor): manufacturer_id = None - row = response.fetchone() + row = cursor.fetchone() if len(row) > 0: try: @@ -59,7 +59,7 @@ def reverse_association(apps, schema_editor): # Now extract the "name" for the manufacturer response = cursor.execute(f"SELECT name from company_company where id={manufacturer_id};") - row = response.fetchone() + row = cursor.fetchone() name = row[0] @@ -106,7 +106,7 @@ def associate_manufacturers(apps, schema_editor): cursor = connection.cursor() response = cursor.execute("select id, MPN from part_supplierpart;") - supplier_parts = response.fetchall() + supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! @@ -166,7 +166,7 @@ def associate_manufacturers(apps, schema_editor): # Extract the company back from the database response = cursor.execute(f"select id from company_company where name='{company_name}';") - row = response.fetchone() + row = cursor.fetchone() manufacturer_id = int(row[0]) # Map both names to the same company @@ -318,7 +318,7 @@ def associate_manufacturers(apps, schema_editor): # Extract all SupplierPart objects from the database cursor = connection.cursor() response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") - results = response.fetchall() + results = cursor.fetchall() part_count = len(results) diff --git a/InvenTree/company/migrations/0026_auto_20201110_1011.py b/InvenTree/company/migrations/0026_auto_20201110_1011.py index 1202fbe7f7..553eced0fc 100644 --- a/InvenTree/company/migrations/0026_auto_20201110_1011.py +++ b/InvenTree/company/migrations/0026_auto_20201110_1011.py @@ -106,7 +106,7 @@ def reverse_currencies(apps, schema_editor): # For each currency code in use, check if we have a matching Currency object for code in codes_in_use: response = cursor.execute(f"SELECT id, suffix from common_currency where suffix='{code}';") - row = response.fetchone() + row = cursor.fetchone() if row is not None: # A match exists! diff --git a/InvenTree/part/migrations/0056_auto_20201110_1125.py b/InvenTree/part/migrations/0056_auto_20201110_1125.py index 3e4572b518..13a512bdd0 100644 --- a/InvenTree/part/migrations/0056_auto_20201110_1125.py +++ b/InvenTree/part/migrations/0056_auto_20201110_1125.py @@ -106,7 +106,7 @@ def reverse_currencies(apps, schema_editor): # For each currency code in use, check if we have a matching Currency object for code in codes_in_use: response = cursor.execute(f"SELECT id, suffix from common_currency where suffix='{code}';") - row = response.fetchone() + row = cursor.fetchone() if row is not None: # A match exists! From 5e9097b5e09d088a01a6a15016802a56f7882d59 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 13:16:32 +1100 Subject: [PATCH 05/42] PSQL: Upper-case column names *must* be qualified with double-quotes Ref: https://www.xspdf.com/resolution/53039249.html --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index a27f56ce3a..509c45aaa2 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -22,7 +22,7 @@ def reverse_association(apps, schema_editor): cursor = connection.cursor() - response = cursor.execute("select id, MPN from part_supplierpart;") + response = cursor.execute('select id, "MPN" from part_supplierpart;') supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects @@ -105,7 +105,7 @@ def associate_manufacturers(apps, schema_editor): cursor = connection.cursor() - response = cursor.execute("select id, MPN from part_supplierpart;") + response = cursor.execute(f'select id, "MPN" from part_supplierpart;') supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects @@ -317,7 +317,7 @@ def associate_manufacturers(apps, schema_editor): # Extract all SupplierPart objects from the database cursor = connection.cursor() - response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") + response = cursor.execute('select id, "MPN", "SKU", manufacturer_id, manufacturer_name from part_supplierpart;') results = cursor.fetchall() part_count = len(results) From 90bef69a5954cc0b3671a6aa17f2cb90be3e8dea Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 16:58:06 +1100 Subject: [PATCH 06/42] Adds "report snippet" class allowing re-usable report snippets to be uploaded --- InvenTree/report/admin.py | 8 ++- .../report/migrations/0006_reportsnippet.py | 27 ++++++++ InvenTree/report/models.py | 69 +++++++++++++------ 3 files changed, 81 insertions(+), 23 deletions(-) create mode 100644 InvenTree/report/migrations/0006_reportsnippet.py diff --git a/InvenTree/report/admin.py b/InvenTree/report/admin.py index 7d6403f5d9..32340921c3 100644 --- a/InvenTree/report/admin.py +++ b/InvenTree/report/admin.py @@ -3,7 +3,12 @@ from __future__ import unicode_literals from django.contrib import admin -from .models import TestReport, ReportAsset +from .models import ReportSnippet, TestReport, ReportAsset + + +class ReportSnippetAdmin(admin.ModelAdmin): + + list_display = ('name', 'description', 'template') class ReportTemplateAdmin(admin.ModelAdmin): @@ -16,5 +21,6 @@ class ReportAssetAdmin(admin.ModelAdmin): list_display = ('asset', 'description') +admin.site.register(ReportSnippet, ReportSnippetAdmin) admin.site.register(TestReport, ReportTemplateAdmin) admin.site.register(ReportAsset, ReportAssetAdmin) diff --git a/InvenTree/report/migrations/0006_reportsnippet.py b/InvenTree/report/migrations/0006_reportsnippet.py new file mode 100644 index 0000000000..c503678345 --- /dev/null +++ b/InvenTree/report/migrations/0006_reportsnippet.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.7 on 2021-02-03 05:57 + +import django.core.validators +from django.db import migrations, models +import report.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('report', '0005_auto_20210119_0815'), + ] + + operations = [ + migrations.CreateModel( + name='ReportSnippet', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Template name', max_length=100, unique=True, verbose_name='Name')), + ('template', models.FileField(help_text='Report template file', upload_to=report.models.rename_template, validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['html', 'htm', 'tex'])], verbose_name='Template')), + ('description', models.CharField(help_text='Report template description', max_length=250, verbose_name='Description')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index 224f2800d8..6b064999d5 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -77,14 +77,18 @@ class WeasyprintReportMixin(WeasyTemplateResponseMixin): self.pdf_filename = kwargs.get('filename', 'report.pdf') -class ReportTemplateBase(models.Model): +class ReportBase(models.Model): """ - Reporting template model. + Base class for uploading html templates """ + class Meta: + abstract = True + def __str__(self): return "{n} - {d}".format(n=self.name, d=self.description) + def getSubdir(self): return '' @@ -105,6 +109,47 @@ class ReportTemplateBase(models.Model): return template + name = models.CharField( + blank=False, max_length=100, + verbose_name=_('Name'), + help_text=_('Template name'), + unique=True, + ) + + template = models.FileField( + upload_to=rename_template, + verbose_name=_('Template'), + help_text=_("Report template file"), + validators=[FileExtensionValidator(allowed_extensions=['html', 'htm', 'tex'])], + ) + + description = models.CharField( + max_length=250, + verbose_name=_('Description'), + help_text=_("Report template description") + ) + + +class ReportSnippet(ReportBase): + """ + Report template 'snippet' which can be used to make templates + that can then be included in other reports. + + Useful for 'common' template actions, sub-templates, etc + """ + + def getSubdir(self): + return "" + + +class ReportTemplateBase(ReportBase): + """ + Reporting template model. + + Able to be passed context data + + """ + def get_context_data(self, request): """ Supply context data to the template for rendering @@ -147,26 +192,6 @@ class ReportTemplateBase(models.Model): wp = WeasyprintReportMixin(request, self.template_name, **kwargs) return wp.render_to_response(context, **kwargs) - name = models.CharField( - blank=False, max_length=100, - verbose_name=_('Name'), - help_text=_('Template name'), - unique=True, - ) - - template = models.FileField( - upload_to=rename_template, - verbose_name=_('Template'), - help_text=_("Report template file"), - validators=[FileExtensionValidator(allowed_extensions=['html', 'htm', 'tex'])], - ) - - description = models.CharField( - max_length=250, - verbose_name=_('Description'), - help_text=_("Report template description") - ) - enabled = models.BooleanField( default=True, verbose_name=_('Enabled'), From cbadb2a888f20c64f6fbffd37e5829d95fbbb3ca Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 21:54:11 +1100 Subject: [PATCH 07/42] Small refactor, and allow editing of ReportAsset in the admin interface --- InvenTree/report/admin.py | 2 +- InvenTree/report/models.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/InvenTree/report/admin.py b/InvenTree/report/admin.py index 32340921c3..5034934ccf 100644 --- a/InvenTree/report/admin.py +++ b/InvenTree/report/admin.py @@ -18,7 +18,7 @@ class ReportTemplateAdmin(admin.ModelAdmin): class ReportAssetAdmin(admin.ModelAdmin): - list_display = ('asset', 'description') + list_display = ('id', 'asset', 'description') admin.site.register(ReportSnippet, ReportSnippetAdmin) diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index 6b064999d5..ba8672f8c1 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -52,9 +52,7 @@ class TexResponse(HttpResponse): def rename_template(instance, filename): - filename = os.path.basename(filename) - - return os.path.join('report', 'report_template', instance.getSubdir(), filename) + return instance.rename_file(filename) def validate_stock_item_report_filters(filters): @@ -92,6 +90,13 @@ class ReportBase(models.Model): def getSubdir(self): return '' + def rename_file(self, filename): + # Function for renaming uploaded file + + filename = os.path.basename(filename) + + return os.path.join('report', 'report_template', self.getSubdir(), filename) + @property def extension(self): return os.path.splitext(self.template.name)[1].lower() From 1d317b1ecbae39c470357cd7eb5a454e2e5f4f07 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 23:16:00 +1100 Subject: [PATCH 08/42] Add django-test-migrations package --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e4ffe6be75..f0e9fbef4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ django-debug-toolbar==2.2 # Debug / profiling toolbar django-admin-shell==0.1.2 # Python shell for the admin interface django-money==1.1 # Django app for currency management certifi # Certifi is (most likely) installed through one of the requirements above -django-error-report==0.2.0 # Error report viewer for the admin interface +django-error-report==0.2.0 # Error report viewer for the admin interface +django-test-migrations==1.1.0 # Unit testing for database migrations inventree # Install the latest version of the InvenTree API python library From 34dbfe6d28cde72c21f26b93433f61b76e20570a Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 23:16:23 +1100 Subject: [PATCH 09/42] Test troublesome migration 0019 --- .../migrations/0019_auto_20200413_0642.py | 62 ++++++++------ InvenTree/company/test_migrations.py | 80 +++++++++++++++++++ 2 files changed, 119 insertions(+), 23 deletions(-) create mode 100644 InvenTree/company/test_migrations.py diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index 509c45aaa2..2bd059edb9 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -1,14 +1,21 @@ # Generated by Django 2.2.10 on 2020-04-13 06:42 +import sys import os from rapidfuzz import fuzz from django.db import migrations, connection from django.db.utils import OperationalError, ProgrammingError +""" +When this migration is tested by CI, it cannot accept user input. +So a simplified version of the migration is implemented. +""" +TESTING = 'test' in sys.argv def clear(): - os.system('cls' if os.name == 'nt' else 'clear') + if not TESTING: + os.system('cls' if os.name == 'nt' else 'clear') def reverse_association(apps, schema_editor): @@ -222,23 +229,31 @@ def associate_manufacturers(apps, schema_editor): clear() # Present a list of options - print("----------------------------------") + if not TESTING: + print("----------------------------------") + print("Checking part [{pk}] ({idx} of {total})".format(pk=part_id, idx=idx+1, total=total)) - print("Manufacturer name: '{n}'".format(n=name)) - print("----------------------------------") - print("Select an option from the list below:") + + if not TESTING: + print("Manufacturer name: '{n}'".format(n=name)) + print("----------------------------------") + print("Select an option from the list below:") - print("0) - Create new manufacturer '{n}'".format(n=name)) - print("") + print("0) - Create new manufacturer '{n}'".format(n=name)) + print("") - for i, m in enumerate(matches[:10]): - print("{i}) - Use manufacturer '{opt}'".format(i=i+1, opt=m)) + for i, m in enumerate(matches[:10]): + print("{i}) - Use manufacturer '{opt}'".format(i=i+1, opt=m)) - print("") - print("OR - Type a new custom manufacturer name") + print("") + print("OR - Type a new custom manufacturer name") while True: - response = str(input("> ")).strip() + if TESTING: + # When running unit tests, simply select the name of the part + response = '0' + else: + response = str(input("> ")).strip() # Attempt to parse user response as an integer try: @@ -300,18 +315,19 @@ def associate_manufacturers(apps, schema_editor): print("") clear() - print("---------------------------------------") - print("The SupplierPart model needs to be migrated,") - print("as the new 'manufacturer' field maps to a 'Company' reference.") - print("The existing 'manufacturer_name' field will be used to match") - print("against possible companies.") - print("This process requires user input.") - print("") - print("Note: This process MUST be completed to migrate the database.") - print("---------------------------------------") - print("") + if not TESTING: + print("---------------------------------------") + print("The SupplierPart model needs to be migrated,") + print("as the new 'manufacturer' field maps to a 'Company' reference.") + print("The existing 'manufacturer_name' field will be used to match") + print("against possible companies.") + print("This process requires user input.") + print("") + print("Note: This process MUST be completed to migrate the database.") + print("---------------------------------------") + print("") - input("Press to continue.") + input("Press to continue.") clear() diff --git a/InvenTree/company/test_migrations.py b/InvenTree/company/test_migrations.py new file mode 100644 index 0000000000..16d9059f23 --- /dev/null +++ b/InvenTree/company/test_migrations.py @@ -0,0 +1,80 @@ +""" +Tests for the company model database migrations +""" + +from django_test_migrations.contrib.unittest_case import MigratorTestCase + + +class TestManufacturerField(MigratorTestCase): + """ + Tests for migration 0019 which migrates from old 'manufacturer_name' field to new 'manufacturer' field + """ + + migrate_from = ('company', '0018_supplierpart_manufacturer') + migrate_to = ('company', '0019_auto_20200413_0642') + + def prepare(self): + """ + Prepare the database by adding some test data 'before' the change: + + - Part object + - Company object (supplier) + - SupplierPart object + """ + + Part = self.old_state.apps.get_model('part', 'part') + Company = self.old_state.apps.get_model('company', 'company') + SupplierPart = self.old_state.apps.get_model('company', 'supplierpart') + + # Create an initial part + part = Part.objects.create( + name='Screw', + description='A single screw' + ) + + # Create a company to act as the supplier + supplier = Company.objects.create( + name='Supplier', + description='A supplier of parts', + is_supplier=True, + is_customer=False, + ) + + # Add some SupplierPart objects + SupplierPart.objects.create( + part=part, + supplier=supplier, + SKU='SCREW.001', + manufacturer_name='ACME', + ) + + SupplierPart.objects.create( + part=part, + supplier=supplier, + SKU='SCREW.002', + manufacturer_name='Zero Corp' + ) + + self.assertEqual(Company.objects.count(), 1) + + def test_company_objects(self): + """ + Test that the new companies have been created successfully + """ + + # Two additional company objects should have been created + Company = self.new_state.apps.get_model('company', 'company') + self.assertEqual(Company.objects.count(), 3) + + # The new company/ies must be marked as "manufacturers" + acme = Company.objects.get(name='ACME') + self.assertTrue(acme.is_manufacturer) + + SupplierPart = self.new_state.apps.get_model('company', 'supplierpart') + parts = SupplierPart.objects.filter(manufacturer=acme) + self.assertEqual(parts.count(), 1) + part = parts.first() + + # Checks on the SupplierPart object + self.assertEqual(part.manufacturer_name, 'ACME') + self.assertEqual(part.manufacturer.name, 'ACME') From bd9447d9aac63d0dc8d78aad24691ea9ffb263cd Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 23:29:14 +1100 Subject: [PATCH 10/42] Add django-migration-linter to ensure django migrations are tippy-top --- InvenTree/InvenTree/settings.py | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 06908e76bf..9587b09d4a 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -213,6 +213,7 @@ INSTALLED_APPS = [ 'djmoney', # django-money integration 'djmoney.contrib.exchange', # django-money exchange rates 'error_report', # Error reporting in the admin interface + 'django_migration_linter', # Linting checking for migration files ] MIDDLEWARE = CONFIG.get('middleware', [ diff --git a/requirements.txt b/requirements.txt index f0e9fbef4d..8c33b7d37e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,5 +31,6 @@ django-money==1.1 # Django app for currency management certifi # Certifi is (most likely) installed through one of the requirements above django-error-report==0.2.0 # Error report viewer for the admin interface django-test-migrations==1.1.0 # Unit testing for database migrations +django-migration-linter==2.5.0 # Linting checks for database migrations inventree # Install the latest version of the InvenTree API python library From c2b5d96186d7889a49771f9c92f2724898d75de0 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 23:39:43 +1100 Subject: [PATCH 11/42] Ensure migration files are covered in coverage tests --- .coveragerc | 2 -- 1 file changed, 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 409c378cac..dbfd8d176d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,6 @@ [run] source = ./InvenTree omit = - # Do not run coverage on migration files - */migrations/* InvenTree/manage.py InvenTree/setup.py InvenTree/InvenTree/middleware.py From f135f11564fa922931786f7f35a1b97600fe2f5d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 23:56:59 +1100 Subject: [PATCH 12/42] Run lint checks on migration files --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 52d0ef1c5c..071b50fec5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,11 @@ script: - rm inventree_default_db.sqlite3 - invoke migrate - invoke import-records -f data.json + # Run linting checks on migration files (django-migration-linter) + # Run subset of linting checks on *ALL* migration files + - cd InvenTree && python manage.py lintmigrations --exclude-migration-test NOT_NULL ADD_UNIQUE -q ok ignore --no-cache && cd .. + # Run stricter checks on *NEW* migrations (old ones are what they are) + - cd InvenTree && python manage.py lintmigrations 79ddea50f507e34195bad635008419daac0d7a5f -q ok ignore --no-cache && cd .. after_success: - coveralls \ No newline at end of file From 5c8e65c2856a283de5d1537f0a8f7f02d9683391 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 00:01:16 +1100 Subject: [PATCH 13/42] Only run linter checks for *new* migration files --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 071b50fec5..872ef0eb0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,8 +50,7 @@ script: - invoke import-records -f data.json # Run linting checks on migration files (django-migration-linter) # Run subset of linting checks on *ALL* migration files - - cd InvenTree && python manage.py lintmigrations --exclude-migration-test NOT_NULL ADD_UNIQUE -q ok ignore --no-cache && cd .. - # Run stricter checks on *NEW* migrations (old ones are what they are) + # Run strict migration file checks on *NEW* migrations (old ones are what they are) - cd InvenTree && python manage.py lintmigrations 79ddea50f507e34195bad635008419daac0d7a5f -q ok ignore --no-cache && cd .. after_success: From 29bb735dc4748d4f03e30a4f8776d1063c688e80 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 00:25:00 +1100 Subject: [PATCH 14/42] Helper functions to automatically extract migration file info --- InvenTree/InvenTree/helpers.py | 64 ++++++++++++++++++++++++++++ InvenTree/company/test_migrations.py | 27 ++++++++++++ 2 files changed, 91 insertions(+) diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 99dc255dac..5cb6cc18dd 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -492,3 +492,67 @@ def addUserPermissions(user, permissions): for permission in permissions: addUserPermission(user, permission) + + +def getMigrationFileNames(app): + """ + Return a list of all migration filenames for provided app + """ + + local_dir = os.path.dirname(os.path.abspath(__file__)) + + migration_dir = os.path.join(local_dir, '..', app, 'migrations') + + files = os.listdir(migration_dir) + + # Regex pattern for migration files + pattern = r"^[\d]+_.*\.py$" + + migration_files = [] + + for f in files: + if re.match(pattern, f): + migration_files.append(f) + + return migration_files + + +def getOldestMigrationFile(app, exclude_extension=True): + """ + Return the filename associated with the oldest migration + """ + + oldest_num = -1 + oldest_file = None + + for f in getMigrationFileNames(app): + num = int(f.split('_')[0]) + + if oldest_file is None or num < oldest_num: + oldest_num = num + oldest_file = f + + if exclude_extension: + oldest_file = oldest_file.replace('.py', '') + + return oldest_file + +def getNewestMigrationFile(app, exclude_extension=True): + """ + Return the filename associated with the newest migration + """ + + newest_file = None + newest_num = -1 + + for f in getMigrationFileNames(app): + num = int(f.split('_')[0]) + + if newest_file is None or num > newest_num: + newest_num = num + newest_file = f + + if exclude_extension: + newest_file = newest_file.replace('.py', '') + + return newest_file diff --git a/InvenTree/company/test_migrations.py b/InvenTree/company/test_migrations.py index 16d9059f23..8ff2cc2a8b 100644 --- a/InvenTree/company/test_migrations.py +++ b/InvenTree/company/test_migrations.py @@ -4,6 +4,33 @@ Tests for the company model database migrations from django_test_migrations.contrib.unittest_case import MigratorTestCase +from InvenTree import helpers + + +class TestForwardMigrations(MigratorTestCase): + + migrate_from = ('company', helpers.getOldestMigrationFile('company')) + migrate_to = ('company', helpers.getNewestMigrationFile('company')) + + def prepare(self): + """ + Create some simple Company data, and ensure that it migrates OK + """ + + Company = self.old_state.apps.get_model('company', 'company') + + Company.objects.create( + name='MSPC', + description='Michael Scotts Paper Company', + is_supplier=True + ) + + def test_migrations(self): + + Company = self.new_state.apps.get_model('company', 'company') + + self.assertEqual(Company.objects.count(), 1) + class TestManufacturerField(MigratorTestCase): """ From e417ff2b4dd06e2dc83384109d2a8612f221e168 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 00:44:37 +1100 Subject: [PATCH 15/42] Test migrations for build app --- InvenTree/InvenTree/helpers.py | 6 +- .../build/migrations/0018_build_reference.py | 2 +- InvenTree/build/test_migrations.py | 118 ++++++++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 InvenTree/build/test_migrations.py diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 5cb6cc18dd..33136d76ce 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -517,7 +517,7 @@ def getMigrationFileNames(app): return migration_files -def getOldestMigrationFile(app, exclude_extension=True): +def getOldestMigrationFile(app, exclude_extension=True, ignore_initial=True): """ Return the filename associated with the oldest migration """ @@ -526,6 +526,10 @@ def getOldestMigrationFile(app, exclude_extension=True): oldest_file = None for f in getMigrationFileNames(app): + + if ignore_initial and f.startswith('0001_initial'): + continue + num = int(f.split('_')[0]) if oldest_file is None or num < oldest_num: diff --git a/InvenTree/build/migrations/0018_build_reference.py b/InvenTree/build/migrations/0018_build_reference.py index bcf4f9b9d4..e8fd938b6a 100644 --- a/InvenTree/build/migrations/0018_build_reference.py +++ b/InvenTree/build/migrations/0018_build_reference.py @@ -9,7 +9,7 @@ def add_default_reference(apps, schema_editor): Best we can do is use the PK of the build order itself. """ - Build = apps.get_model('build', 'Build') + Build = apps.get_model('build', 'build') count = 0 diff --git a/InvenTree/build/test_migrations.py b/InvenTree/build/test_migrations.py new file mode 100644 index 0000000000..1e95cfb54e --- /dev/null +++ b/InvenTree/build/test_migrations.py @@ -0,0 +1,118 @@ +""" +Tests for the build model database migrations +""" + +from django_test_migrations.contrib.unittest_case import MigratorTestCase + +from InvenTree import helpers + + +class TestForwardMigrations(MigratorTestCase): + """ + Test entire schema migration sequence for the build app + """ + + migrate_from = ('build', helpers.getOldestMigrationFile('build')) + migrate_to = ('build', helpers.getNewestMigrationFile('build')) + + def prepare(self): + """ + Create initial data! + """ + + Part = self.old_state.apps.get_model('part', 'part') + + buildable_part = Part.objects.create( + name='Widget', + description='Buildable Part', + active=True, + ) + + with self.assertRaises(TypeError): + # Cannot set the 'assembly' field as it hasn't been added to the db schema + Part.objects.create( + name='Blorb', + description='ABCDE', + assembly=True + ) + + Build = self.old_state.apps.get_model('build', 'build') + + Build.objects.create( + part=buildable_part, + title='A build of some stuff', + quantity=50 + ) + + def test_items_exist(self): + + Part = self.new_state.apps.get_model('part', 'part') + + self.assertEqual(Part.objects.count(), 1) + + Build = self.new_state.apps.get_model('build', 'build') + + self.assertEqual(Build.objects.count(), 1) + + # Check that the part object now has an assembly field + part = Part.objects.all().first() + part.assembly = True + part.save() + part.assembly = False + part.save() + + +class TestReferenceMigration(MigratorTestCase): + """ + Test custom migration which adds 'reference' field to Build model + """ + + migrate_from = ('build', helpers.getOldestMigrationFile('build')) + migrate_to = ('build', '0018_build_reference') + + def prepare(self): + """ + Create some builds + """ + + Part = self.old_state.apps.get_model('part', 'part') + + part = Part.objects.create( + name='Part', + description='A test part' + ) + + Build = self.old_state.apps.get_model('build', 'build') + + Build.objects.create( + part=part, + title='My very first build', + quantity=10 + ) + + Build.objects.create( + part=part, + title='My very second build', + quantity=10 + ) + + Build.objects.create( + part=part, + title='My very third build', + quantity=10 + ) + + # Ensure that the builds *do not* have a 'reference' field + for build in Build.objects.all(): + with self.assertRaises(AttributeError): + print(build.reference) + + def test_build_reference(self): + + Build = self.new_state.apps.get_model('build', 'build') + + self.assertEqual(Build.objects.count(), 3) + + # Check that the build reference is properly assigned + for build in Build.objects.all(): + self.assertEqual(str(build.reference), str(build.pk)) From 75431f0ee4f7fff9e1a1653cf45fb1562803c945 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 00:51:00 +1100 Subject: [PATCH 16/42] Flake errors --- InvenTree/InvenTree/helpers.py | 1 + setup.cfg | 2 ++ 2 files changed, 3 insertions(+) diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 33136d76ce..62e50bd52f 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -540,6 +540,7 @@ def getOldestMigrationFile(app, exclude_extension=True, ignore_initial=True): oldest_file = oldest_file.replace('.py', '') return oldest_file + def getNewestMigrationFile(app, exclude_extension=True): """ diff --git a/setup.cfg b/setup.cfg index 6e2a44f055..ea232522a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,8 @@ ignore = C901, # - N802 - function name should be lowercase (In the future, we should conform to this!) N802, + # - N806 - variable should be lowercase + N806, N812, exclude = .git,__pycache__,*/migrations/*,*/lib/*,*/bin/*,*/media/*,*/static/*,*ci_*.py* max-complexity = 20 From b284fe7f2b706ff682684a14fcfe4bb3e8365b75 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 15:15:49 +1100 Subject: [PATCH 17/42] Remove quotes around column names (cherry picked from commit 386cb2dd3ac03de49b037e171109cf124001a030) --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index 2bd059edb9..fe35311201 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -165,7 +165,7 @@ def associate_manufacturers(apps, schema_editor): # Manually create a new database row # Note: Have to fill out all empty string values! - new_manufacturer_query = f"insert into company_company ('name', 'description', 'is_customer', 'is_supplier', 'is_manufacturer', 'address', 'website', 'phone', 'email', 'contact', 'link', 'notes') values ('{company_name}', '{company_name}', false, false, true, '', '', '', '', '', '', '');" + new_manufacturer_query = f"insert into company_company (name, description, is_customer, is_supplier, is_manufacturer, address, website, phone, email, contact, link, notes) values ('{company_name}', '{company_name}', false, false, true, '', '', '', '', '', '', '');" cursor = connection.cursor() From ad0b59bf11015349876e5d67a08b3afd803ff920 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 15:19:28 +1100 Subject: [PATCH 18/42] Bug fxi (cherry picked from commit 0e11b722be67175ad4d8cb80ae703441201a186a) --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index fe35311201..c07265f9eb 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -151,7 +151,7 @@ def associate_manufacturers(apps, schema_editor): # Have we already mapped this if name in links.keys(): - print(" - Part[{pk}]: Mapped '{n}' - '{c}'".format(pk=part_id, n=name, c=links[name].name)) + print(" - Part[{pk}]: Mapped '{n}' - manufacturer <{c}>".format(pk=part_id, n=name, c=links[name])) query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" result = query.execute() From 93f0dbd4eecc849e6498792543578bcf41d7d163 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 15:56:49 +1100 Subject: [PATCH 19/42] Bug fix: add missing line (cherry picked from commit 2303e03580d1b34976ac93c124d33d3aa32da238) --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index c07265f9eb..053bd62360 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -153,6 +153,8 @@ def associate_manufacturers(apps, schema_editor): if name in links.keys(): print(" - Part[{pk}]: Mapped '{n}' - manufacturer <{c}>".format(pk=part_id, n=name, c=links[name])) + manufacturer_id = links[name] + query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" result = query.execute() return True From d811f3c48a80ec703f1f457e41e6f11c0a0156b4 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 15:57:15 +1100 Subject: [PATCH 20/42] Typo fix (cherry picked from commit c58399206c523b88594d63d5e696f6468639e466) --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index 053bd62360..1e1bcda922 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -156,7 +156,7 @@ def associate_manufacturers(apps, schema_editor): manufacturer_id = links[name] query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" - result = query.execute() + result = cursor.execute(query) return True # Mapping not possible From e407b99d0d1f89bab3d90fa519631b0088735f4e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 09:13:11 +1100 Subject: [PATCH 21/42] Add initial migration unit test for the 'part' app --- InvenTree/part/test_migrations.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 InvenTree/part/test_migrations.py diff --git a/InvenTree/part/test_migrations.py b/InvenTree/part/test_migrations.py new file mode 100644 index 0000000000..0dbcdb1af8 --- /dev/null +++ b/InvenTree/part/test_migrations.py @@ -0,0 +1,52 @@ +""" +Unit tests for the part model database migrations +""" + +from django_test_migrations.contrib.unittest_case import MigratorTestCase + +from InvenTree import helpers + + +class TestForwardMigrations(MigratorTestCase): + """ + Test entire schema migration sequence for the part app + """ + + migrate_from = ('part', helpers.getOldestMigrationFile('part')) + migrate_to = ('part', helpers.getNewestMigrationFile('part')) + + def prepare(self): + """ + Create initial data + """ + + Part = self.old_state.apps.get_model('part', 'part') + + Part.objects.create(name='A', description='My part A') + Part.objects.create(name='B', description='My part B') + Part.objects.create(name='C', description='My part C') + Part.objects.create(name='D', description='My part D') + Part.objects.create(name='E', description='My part E') + + # Extract one part object to investigate + p = Part.objects.all().last() + + # Initially some fields are not present + with self.assertRaises(AttributeError): + print(p.has_variants) + + with self.assertRaises(AttributeError): + print(p.is_template) + + + def test_models_exist(self): + + Part = self.new_state.apps.get_model('part', 'part') + + self.assertEqual(Part.objects.count(), 5) + + for part in Part.objects.all(): + part.is_template = True + part.save() + part.is_template = False + part.save() From cabac6614cbe9f11ce711dce8f389c4131e05f46 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 09:13:23 +1100 Subject: [PATCH 22/42] Add unit test for currency migration --- InvenTree/company/test_migrations.py | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/InvenTree/company/test_migrations.py b/InvenTree/company/test_migrations.py index 8ff2cc2a8b..1a51a5a5b0 100644 --- a/InvenTree/company/test_migrations.py +++ b/InvenTree/company/test_migrations.py @@ -105,3 +105,67 @@ class TestManufacturerField(MigratorTestCase): # Checks on the SupplierPart object self.assertEqual(part.manufacturer_name, 'ACME') self.assertEqual(part.manufacturer.name, 'ACME') + + +class TestCurrencyMigration(MigratorTestCase): + """ + Tests for upgrade from basic currency support to django-money + """ + + migrate_from = ('company', '0025_auto_20201110_1001') + migrate_to = ('company', '0026_auto_20201110_1011') + + def prepare(self): + """ + Prepare some data: + + - A part to buy + - A supplier to buy from + - A supplier part + - Multiple currency objects + - Multiple supplier price breaks + """ + + Part = self.old_state.apps.get_model('part', 'part') + + part = Part.objects.create( + name="PART", description="A purchaseable part", + purchaseable=True, + level=0, + tree_id=0, + lft=0, + rght=0 + ) + + Company = self.old_state.apps.get_model('company', 'company') + + supplier = Company.objects.create(name='Supplier', description='A supplier', is_supplier=True) + + SupplierPart = self.old_state.apps.get_model('company', 'supplierpart') + + sp = SupplierPart.objects.create(part=part, supplier=supplier, SKU='12345') + + Currency = self.old_state.apps.get_model('common', 'currency') + + aud = Currency.objects.create(symbol='$', suffix='AUD', description='Australian Dollars', value=1.0) + usd = Currency.objects.create(symbol='$', suffix='USD', description='US Dollars', value=1.0) + + PB = self.old_state.apps.get_model('company', 'supplierpricebreak') + + PB.objects.create(part=sp, quantity=10, cost=5, currency=aud) + PB.objects.create(part=sp, quantity=20, cost=3, currency=aud) + PB.objects.create(part=sp, quantity=30, cost=2, currency=aud) + + PB.objects.create(part=sp, quantity=40, cost=2, currency=usd) + PB.objects.create(part=sp, quantity=50, cost=2, currency=usd) + + for pb in PB.objects.all(): + self.assertIsNone(pb.price) + + def test_currency_migration(self): + + PB = self.new_state.apps.get_model('company', 'supplierpricebreak') + + for pb in PB.objects.all(): + # Test that a price has been assigned + self.assertIsNotNone(pb.price) From b107c54eb252deb9795f0f19fd383792ddb19720 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 09:13:45 +1100 Subject: [PATCH 23/42] PEP fix --- InvenTree/part/test_migrations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InvenTree/part/test_migrations.py b/InvenTree/part/test_migrations.py index 0dbcdb1af8..41fead4b30 100644 --- a/InvenTree/part/test_migrations.py +++ b/InvenTree/part/test_migrations.py @@ -38,7 +38,6 @@ class TestForwardMigrations(MigratorTestCase): with self.assertRaises(AttributeError): print(p.is_template) - def test_models_exist(self): Part = self.new_state.apps.get_model('part', 'part') From 3ccc500e8ec10f6bdd94794d857f121a2d1ba8ff Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 12:58:19 +1100 Subject: [PATCH 24/42] Add more context data to report --- InvenTree/report/models.py | 16 ++++++++++++++-- InvenTree/templates/js/report.js | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index ba8672f8c1..c2b5d729fe 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -175,6 +175,10 @@ class ReportTemplateBase(ReportBase): context = self.get_context_data(request) + context['media'] = settings.MEDIA_ROOT + + context['report_name'] = self.name + context['report_description'] = self.description context['request'] = request context['user'] = request.user context['datetime'] = datetime.datetime.now() @@ -194,8 +198,16 @@ class ReportTemplateBase(ReportBase): raise ValidationError("Enable LaTeX support in config.yaml") elif self.extension in ['.htm', '.html']: # Render HTML template to PDF - wp = WeasyprintReportMixin(request, self.template_name, **kwargs) - return wp.render_to_response(context, **kwargs) + wp = WeasyprintReportMixin( + request, + self.template_name, + base_url=request.build_absolute_uri("/"), + presentational_hints=True, + **kwargs) + + return wp.render_to_response( + context, + **kwargs) enabled = models.BooleanField( default=True, diff --git a/InvenTree/templates/js/report.js b/InvenTree/templates/js/report.js index da84433bac..6e47706917 100644 --- a/InvenTree/templates/js/report.js +++ b/InvenTree/templates/js/report.js @@ -10,6 +10,15 @@ function selectTestReport(reports, items, options={}) { * (via AJAX) from the server. */ + // If there is only a single report available, just print! + if (reports.length == 1) { + if (options.success) { + options.success(reports[0].pk); + } + + return; + } + var modal = options.modal || '#modal-form'; var report_list = makeOptionsList( From 801b945438e112b4727be31606ab211aec6e6274 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 13:33:14 +1100 Subject: [PATCH 25/42] Add current date to report context --- InvenTree/report/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index c2b5d729fe..56d06090a1 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -181,6 +181,7 @@ class ReportTemplateBase(ReportBase): context['report_description'] = self.description context['request'] = request context['user'] = request.user + context['date'] = datetime.datetime.now().date() context['datetime'] = datetime.datetime.now() if self.extension == '.tex': From 6230fb3614d8555cfbb0c60b917f4f37d8b29a0f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 13:41:47 +1100 Subject: [PATCH 26/42] Add custom report template tags --- InvenTree/part/templatetags/report.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 InvenTree/part/templatetags/report.py diff --git a/InvenTree/part/templatetags/report.py b/InvenTree/part/templatetags/report.py new file mode 100644 index 0000000000..32eec76db0 --- /dev/null +++ b/InvenTree/part/templatetags/report.py @@ -0,0 +1,22 @@ +""" +Custom template tags for report generation +""" + +import os + +from django import template +from django.conf import settings + +register = template.Library() + + +@register.simple_tag() +def asset(filename): + """ + Return fully-qualified path for an upload report asset file. + """ + + path = os.path.join(settings.MEDIA_ROOT, 'report', 'assets', filename) + path = os.path.abspath(path) + + return f"file://{path}" From cf0c43d89938df092f3bbb66b13f0aa3000a6a73 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 13:54:26 +1100 Subject: [PATCH 27/42] Add report base template --- .../report/inventree_report_base.html | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 InvenTree/report/templates/report/inventree_report_base.html diff --git a/InvenTree/report/templates/report/inventree_report_base.html b/InvenTree/report/templates/report/inventree_report_base.html new file mode 100644 index 0000000000..936a9758a9 --- /dev/null +++ b/InvenTree/report/templates/report/inventree_report_base.html @@ -0,0 +1,51 @@ +{% load report %} + + + + + + + +
+ {% block header_content %} + {% endblock %} +
+ +
+ {% block page_content %} + {% endblock %} +
+ +
+ {% block footer_content %} + {% endblock %} +
+ + \ No newline at end of file From ddbf2a6313def4f9c43ac894545d4e70f6541214 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 14:49:11 +1100 Subject: [PATCH 28/42] Add margin callouts for report template base --- .../report/inventree_report_base.html | 79 +++++++++++++++---- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/InvenTree/report/templates/report/inventree_report_base.html b/InvenTree/report/templates/report/inventree_report_base.html index 936a9758a9..afcd31f2ed 100644 --- a/InvenTree/report/templates/report/inventree_report_base.html +++ b/InvenTree/report/templates/report/inventree_report_base.html @@ -3,29 +3,76 @@ @@ -33,17 +80,17 @@ -
+
{% block header_content %} {% endblock %}
-
+
{% block page_content %} {% endblock %}
-
+
{% block footer_content %} {% endblock %}
From a025b7239d3e564c52d0cb1f945e4145b6de0f2d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 15:29:46 +1100 Subject: [PATCH 29/42] Adds simple test-report template --- InvenTree/part/templatetags/report.py | 22 ---- .../report/inventree_report_base.html | 3 +- .../report/inventree_test_report.html | 105 ++++++++++++++++++ InvenTree/report/templatetags/report.py | 54 +++++++++ 4 files changed, 161 insertions(+), 23 deletions(-) delete mode 100644 InvenTree/part/templatetags/report.py create mode 100644 InvenTree/report/templates/report/inventree_test_report.html create mode 100644 InvenTree/report/templatetags/report.py diff --git a/InvenTree/part/templatetags/report.py b/InvenTree/part/templatetags/report.py deleted file mode 100644 index 32eec76db0..0000000000 --- a/InvenTree/part/templatetags/report.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Custom template tags for report generation -""" - -import os - -from django import template -from django.conf import settings - -register = template.Library() - - -@register.simple_tag() -def asset(filename): - """ - Return fully-qualified path for an upload report asset file. - """ - - path = os.path.join(settings.MEDIA_ROOT, 'report', 'assets', filename) - path = os.path.abspath(path) - - return f"file://{path}" diff --git a/InvenTree/report/templates/report/inventree_report_base.html b/InvenTree/report/templates/report/inventree_report_base.html index afcd31f2ed..c46ddb32e6 100644 --- a/InvenTree/report/templates/report/inventree_report_base.html +++ b/InvenTree/report/templates/report/inventree_report_base.html @@ -5,7 +5,6 @@ @page { size: A4; margin: 2cm; - margin-top: 4cm; font-family: Arial, Helvetica, sans-serif; font-size: 75%; @@ -46,10 +45,12 @@ } .header { + {% block header_style %} top: 0px; left: 0px; position: fixed; width: 100%; + {% endblock %} margin-top: -2.5cm; } diff --git a/InvenTree/report/templates/report/inventree_test_report.html b/InvenTree/report/templates/report/inventree_test_report.html new file mode 100644 index 0000000000..7c99eec6ab --- /dev/null +++ b/InvenTree/report/templates/report/inventree_test_report.html @@ -0,0 +1,105 @@ +{% extends "report/inventree_report_base.html" %} + +{% load i18n %} +{% load report %} +{% load inventree_extras %} + +{% block style %} +.test-table { + width: 100%; +} + +{% block bottom_left %} +content: "{{ date.isoformat }}"; +{% endblock %} + +{% block bottom_center %} +content: "InvenTree v{% inventree_version %}"; +{% endblock %} + +{% block top_center %} +content: "{% trans 'Stock Item Test Report' %}"; +{% endblock %} + +.test-row { + padding: 3px; +} + +.test-pass { + color: #5f5; +} + +.test-fail { + color: #F55; +} + +.container { + padding: 5px; + border: 1px solid; +} + +.text-left { + display: inline-block; + width: 50%; +} + +.img-right { + display: inline; + align-content: right; + align-items: right; + width: 50%; +} + +{% endblock %} + +{% block page_content %} + +
+
+

+ {{ stock_item.part.full_name }} +

+
+
+ +
+

+ {% trans "Serial Number" %}: {{ stock_item.serial }} +

+
+
+ +

{% trans "Test Results" %}

+ + + + + + + + + + + + + + + + {% for test in result_list %} + + + {% if test.result %} + + {% else %} + + {% endif %} + + + + + {% endfor %} + + +
{% trans "Test" %}{% trans "Result" %}{% trans "Value" %}{% trans "User" %}{% trans "Date" %}
{{ test.test }}{% trans "Pass" %}{% trans "Fail" %}{{ test.value }}{{ test.user.username }}{{ test.date.date.isoformat }}
+ +{% endblock %} \ No newline at end of file diff --git a/InvenTree/report/templatetags/report.py b/InvenTree/report/templatetags/report.py new file mode 100644 index 0000000000..1867ffe032 --- /dev/null +++ b/InvenTree/report/templatetags/report.py @@ -0,0 +1,54 @@ +""" +Custom template tags for report generation +""" + +import os + +from django import template +from django.conf import settings + +from part.models import Part +from stock.models import StockItem + +register = template.Library() + + +@register.simple_tag() +def asset(filename): + """ + Return fully-qualified path for an upload report asset file. + """ + + path = os.path.join(settings.MEDIA_ROOT, 'report', 'assets', filename) + path = os.path.abspath(path) + + return f"file://{path}" + + +@register.simple_tag() +def part_image(part): + """ + Return a fully-qualified path for a part image + """ + + if type(part) is Part: + img = part.image.name + + elif type(part) is StockItem: + img = part.part.image.name + + else: + img = '' + + path = os.path.join(settings.MEDIA_ROOT, img) + path = os.path.abspath(path) + + print("Path:", path) + + if not os.path.exists(path) or not os.path.isfile(path): + # Image does not exist + # Return the 'blank' image + path = os.path.join(settings.STATIC_ROOT, 'img', 'blank_image.png') + path = os.path.abspath(path) + + return f"file://{path}" From 6ebebe03d4e8ed464fdb38f00bc4ee11fa2d1cb3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 15:38:41 +1100 Subject: [PATCH 30/42] Simplify report snippet class definition --- InvenTree/report/admin.py | 10 +++--- .../report/migrations/0006_reportsnippet.py | 10 ++---- InvenTree/report/models.py | 36 ++++++++++++------- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/InvenTree/report/admin.py b/InvenTree/report/admin.py index 5034934ccf..390964b590 100644 --- a/InvenTree/report/admin.py +++ b/InvenTree/report/admin.py @@ -6,16 +6,16 @@ from django.contrib import admin from .models import ReportSnippet, TestReport, ReportAsset -class ReportSnippetAdmin(admin.ModelAdmin): - - list_display = ('name', 'description', 'template') - - class ReportTemplateAdmin(admin.ModelAdmin): list_display = ('name', 'description', 'template', 'filters', 'enabled') +class ReportSnippetAdmin(admin.ModelAdmin): + + list_display = ('id', 'snippet', 'description') + + class ReportAssetAdmin(admin.ModelAdmin): list_display = ('id', 'asset', 'description') diff --git a/InvenTree/report/migrations/0006_reportsnippet.py b/InvenTree/report/migrations/0006_reportsnippet.py index c503678345..6875ebb530 100644 --- a/InvenTree/report/migrations/0006_reportsnippet.py +++ b/InvenTree/report/migrations/0006_reportsnippet.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.7 on 2021-02-03 05:57 +# Generated by Django 3.0.7 on 2021-02-04 04:37 import django.core.validators from django.db import migrations, models @@ -16,12 +16,8 @@ class Migration(migrations.Migration): name='ReportSnippet', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(help_text='Template name', max_length=100, unique=True, verbose_name='Name')), - ('template', models.FileField(help_text='Report template file', upload_to=report.models.rename_template, validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['html', 'htm', 'tex'])], verbose_name='Template')), - ('description', models.CharField(help_text='Report template description', max_length=250, verbose_name='Description')), + ('snippet', models.FileField(help_text='Report snippet file', upload_to=report.models.rename_snippet, validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['html', 'htm'])])), + ('description', models.CharField(help_text='Snippet file description', max_length=250)), ], - options={ - 'abstract': False, - }, ), ] diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index 56d06090a1..1c9db7bbb9 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -135,18 +135,6 @@ class ReportBase(models.Model): ) -class ReportSnippet(ReportBase): - """ - Report template 'snippet' which can be used to make templates - that can then be included in other reports. - - Useful for 'common' template actions, sub-templates, etc - """ - - def getSubdir(self): - return "" - - class ReportTemplateBase(ReportBase): """ Reporting template model. @@ -264,6 +252,30 @@ class TestReport(ReportTemplateBase): } +def rename_snippet(instance, filename): + + filename = os.path.basename(filename) + + return os.path.join('report', 'snippets', filename) + + +class ReportSnippet(models.Model): + """ + Report template 'snippet' which can be used to make templates + that can then be included in other reports. + + Useful for 'common' template actions, sub-templates, etc + """ + + snippet = models.FileField( + upload_to=rename_snippet, + help_text=_('Report snippet file'), + validators=[FileExtensionValidator(allowed_extensions=['html', 'htm'])], + ) + + description = models.CharField(max_length=250, help_text=_("Snippet file description")) + + def rename_asset(instance, filename): filename = os.path.basename(filename) From 0de284f1aa8594e4703813c14f4eab6a345563be Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Feb 2021 16:06:01 +1100 Subject: [PATCH 31/42] Unit fixes --- InvenTree/report/models.py | 1 - .../report/templates/report/inventree_report_base.html | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index 1c9db7bbb9..6ca0ac1dcd 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -86,7 +86,6 @@ class ReportBase(models.Model): def __str__(self): return "{n} - {d}".format(n=self.name, d=self.description) - def getSubdir(self): return '' diff --git a/InvenTree/report/templates/report/inventree_report_base.html b/InvenTree/report/templates/report/inventree_report_base.html index c46ddb32e6..084b100cab 100644 --- a/InvenTree/report/templates/report/inventree_report_base.html +++ b/InvenTree/report/templates/report/inventree_report_base.html @@ -3,8 +3,12 @@