Merge pull request #2155 from SchrodingersGat/natural-sort

Natural sort

InvenTree/InvenTree/filters.py

@@ -34,18 +34,47 @@ class InvenTreeOrderingFilter(OrderingFilter):
             Ordering fields should be mapped to separate fields
             """
 
-            for idx, field in enumerate(ordering):
+            ordering_initial = ordering
+            ordering = []
 
-                reverse = False
+            for field in ordering_initial:
                 
-                if field.startswith('-'):
+                reverse = field.startswith('-')
-                    field = field[1:]
-                    reverse = True
-
-                if field in aliases:
-                    ordering[idx] = aliases[field]
 
                 if reverse:
-                        ordering[idx] = '-' + ordering[idx]
+                    field = field[1:]
+
+                # Are aliases defined for this field?
+                if field in aliases:
+                    alias = aliases[field]
+                else:
+                    alias = field
+
+                """
+                Potentially, a single field could be "aliased" to multiple field,
+                
+                (For example to enforce a particular ordering sequence)
+
+                e.g. to filter first by the integer value...
+
+                ordering_field_aliases = {
+                    "reference": ["integer_ref", "reference"]
+                }
+
+                """
+
+                if type(alias) is str:
+                    alias = [alias]
+                elif type(alias) in [list, tuple]:
+                    pass
+                else:
+                    # Unsupported alias type
+                    continue
+
+                for a in alias:
+                    if reverse:
+                        a = '-' + a
+
+                    ordering.append(a)
 
         return ordering

InvenTree/InvenTree/models.py

@@ -4,6 +4,7 @@ Generic models which provide extra functionality over base Django model types.
 
 from __future__ import unicode_literals
 
+import re
 import os
 import logging
 
@@ -43,6 +44,48 @@ def rename_attachment(instance, filename):
     return os.path.join(instance.getSubdir(), filename)
 
 
+class ReferenceIndexingMixin(models.Model):
+    """
+    A mixin for keeping track of numerical copies of the "reference" field.
+
+    Here, we attempt to convert a "reference" field value (char) to an integer,
+    for performing fast natural sorting.
+
+    This requires extra database space (due to the extra table column),
+    but is required as not all supported database backends provide equivalent casting.
+
+    This mixin adds a field named 'reference_int'.
+
+    - If the 'reference' field can be cast to an integer, it is stored here
+    - If the 'reference' field *starts* with an integer, it is stored here
+    - Otherwise, we store zero
+    """
+
+    class Meta:
+        abstract = True
+
+    def rebuild_reference_field(self):
+
+        reference = getattr(self, 'reference', '')
+
+        # Default value if we cannot convert to an integer
+        ref_int = 0
+
+        # Look at the start of the string - can it be "integerized"?
+        result = re.match(r"^(\d+)", reference)
+
+        if result and len(result.groups()) == 1:
+            ref = result.groups()[0]
+            try:
+                ref_int = int(ref)
+            except:
+                ref_int = 0
+
+        self.reference_int = ref_int
+
+    reference_int = models.IntegerField(default=0)
+
+
 class InvenTreeAttachment(models.Model):
     """ Provides an abstracted class for managing file attachments.
 

InvenTree/build/admin.py

@@ -9,6 +9,10 @@ from .models import Build, BuildItem
 
 class BuildAdmin(ImportExportModelAdmin):
 
+    exclude = [
+        'reference_int',
+    ]
+
     list_display = (
         'reference',
         'title',

InvenTree/build/api.py

@@ -17,6 +17,7 @@ from django_filters import rest_framework as rest_filters
 
 from InvenTree.api import AttachmentMixin
 from InvenTree.helpers import str2bool, isNull
+from InvenTree.filters import InvenTreeOrderingFilter
 from InvenTree.status_codes import BuildStatus
 
 from .models import Build, BuildItem, BuildOrderAttachment
@@ -68,7 +69,7 @@ class BuildList(generics.ListCreateAPIView):
     filter_backends = [
         DjangoFilterBackend,
         filters.SearchFilter,
-        filters.OrderingFilter,
+        InvenTreeOrderingFilter,
     ]
 
     ordering_fields = [
@@ -83,6 +84,10 @@ class BuildList(generics.ListCreateAPIView):
         'responsible',
     ]
 
+    ordering_field_aliases = {
+        'reference': ['reference_int', 'reference'],
+    }
+
     search_fields = [
         'reference',
         'part__name',

InvenTree/build/migrations/0031_build_reference_int.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.5 on 2021-10-14 06:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('build', '0030_alter_build_reference'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='reference_int',
+            field=models.IntegerField(default=0),
+        ),
+    ]

InvenTree/build/migrations/0032_auto_20211014_0632.py

@@ -0,0 +1,50 @@
+# Generated by Django 3.2.5 on 2021-10-14 06:32
+
+import re
+
+from django.db import migrations
+
+
+def build_refs(apps, schema_editor):
+    """
+    Rebuild the integer "reference fields" for existing Build objects
+    """
+
+    BuildOrder = apps.get_model('build', 'build')
+
+    for build in BuildOrder.objects.all():
+
+        ref = 0
+
+        result = re.match(r"^(\d+)", build.reference)
+
+        if result and len(result.groups()) == 1:
+            try:
+                ref = int(result.groups()[0])
+            except:
+                ref = 0
+
+        build.reference_int = ref
+        build.save()
+
+def unbuild_refs(apps, schema_editor):
+    """
+    Provided only for reverse migration compatibility
+    """
+    pass
+
+
+class Migration(migrations.Migration):
+
+    atomic = False
+
+    dependencies = [
+        ('build', '0031_build_reference_int'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            build_refs,
+            reverse_code=unbuild_refs
+        )
+    ]

InvenTree/build/models.py

@@ -28,7 +28,7 @@ from mptt.exceptions import InvalidMove
 from InvenTree.status_codes import BuildStatus, StockStatus, StockHistoryCode
 from InvenTree.helpers import increment, getSetting, normalize, MakeBarcode
 from InvenTree.validators import validate_build_order_reference
-from InvenTree.models import InvenTreeAttachment
+from InvenTree.models import InvenTreeAttachment, ReferenceIndexingMixin
 
 import common.models
 
@@ -69,7 +69,7 @@ def get_next_build_number():
     return reference
 
 
-class Build(MPTTModel):
+class Build(MPTTModel, ReferenceIndexingMixin):
     """ A Build object organises the creation of new StockItem objects from other existing StockItem objects.
 
     Attributes:
@@ -108,6 +108,8 @@ class Build(MPTTModel):
 
     def save(self, *args, **kwargs):
 
+        self.rebuild_reference_field()
+
         try:
             super().save(*args, **kwargs)
         except InvalidMove:

InvenTree/build/test_build.py

@@ -118,6 +118,26 @@ class BuildTest(TestCase):
 
         self.stock_3_1 = StockItem.objects.create(part=self.sub_part_3, quantity=1000)
 
+    def test_ref_int(self):
+        """
+        Test the "integer reference" field used for natural sorting
+        """
+
+        for ii in range(10):
+            build = Build(
+                reference=f"{ii}_abcde",
+                quantity=1,
+                part=self.assembly,
+                title="Making some parts"
+            )
+
+            self.assertEqual(build.reference_int, 0)
+
+            build.save()
+
+            # After saving, the integer reference should have been updated
+            self.assertEqual(build.reference_int, ii)
+
     def test_init(self):
         # Perform some basic tests before we start the ball rolling
 

InvenTree/order/admin.py

@@ -20,6 +20,10 @@ class PurchaseOrderLineItemInlineAdmin(admin.StackedInline):
 
 class PurchaseOrderAdmin(ImportExportModelAdmin):
 
+    exclude = [
+        'reference_int',
+    ]
+
     list_display = (
         'reference',
         'supplier',
@@ -41,6 +45,10 @@ class PurchaseOrderAdmin(ImportExportModelAdmin):
 
 class SalesOrderAdmin(ImportExportModelAdmin):
 
+    exclude = [
+        'reference_int',
+    ]
+
     list_display = (
         'reference',
         'customer',

InvenTree/order/api.py

@@ -152,9 +152,13 @@ class POList(generics.ListCreateAPIView):
     filter_backends = [
         rest_filters.DjangoFilterBackend,
         filters.SearchFilter,
-        filters.OrderingFilter,
+        InvenTreeOrderingFilter,
     ]
 
+    ordering_field_aliases = {
+        'reference': ['reference_int', 'reference'],
+    }
+
     filter_fields = [
         'supplier',
     ]
@@ -504,9 +508,13 @@ class SOList(generics.ListCreateAPIView):
     filter_backends = [
         rest_filters.DjangoFilterBackend,
         filters.SearchFilter,
-        filters.OrderingFilter,
+        InvenTreeOrderingFilter,
     ]
 
+    ordering_field_aliases = {
+        'reference': ['reference_int', 'reference'],
+    }
+
     filter_fields = [
         'customer',
     ]

InvenTree/order/migrations/0051_auto_20211014_0623.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.5 on 2021-10-14 06:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('order', '0050_alter_purchaseorderlineitem_destination'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='purchaseorder',
+            name='reference_int',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='salesorder',
+            name='reference_int',
+            field=models.IntegerField(default=0),
+        ),
+    ]

InvenTree/order/migrations/0052_auto_20211014_0631.py

@@ -0,0 +1,66 @@
+# Generated by Django 3.2.5 on 2021-10-14 06:31
+
+import re
+
+from django.db import migrations
+
+def build_refs(apps, schema_editor):
+    """
+    Rebuild the integer "reference fields" for existing Build objects
+    """
+
+    PurchaseOrder = apps.get_model('order', 'purchaseorder')
+
+    for order in PurchaseOrder.objects.all():
+
+        ref = 0
+
+        result = re.match(r"^(\d+)", order.reference)
+
+        if result and len(result.groups()) == 1:
+            try:
+                ref = int(result.groups()[0])
+            except:
+                ref = 0
+
+        order.reference_int = ref
+        order.save()
+
+    SalesOrder = apps.get_model('order', 'salesorder')
+
+    for order in SalesOrder.objects.all():
+
+        ref = 0
+
+        result = re.match(r"^(\d+)", order.reference)
+
+        if result and len(result.groups()) == 1:
+            try:
+                ref = int(result.groups()[0])
+            except:
+                ref = 0
+
+        order.reference_int = ref
+        order.save()
+
+
+def unbuild_refs(apps, schema_editor):
+    """
+    Provided only for reverse migration compatibility
+    """
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('order', '0051_auto_20211014_0623'),
+    ]
+
+
+    operations = [
+        migrations.RunPython(
+            build_refs,
+            reverse_code=unbuild_refs
+        )
+    ]

InvenTree/order/models.py

@@ -28,7 +28,7 @@ from company.models import Company, SupplierPart
 from InvenTree.fields import InvenTreeModelMoneyField, RoundingDecimalField
 from InvenTree.helpers import decimal2string, increment, getSetting
 from InvenTree.status_codes import PurchaseOrderStatus, SalesOrderStatus, StockStatus, StockHistoryCode
-from InvenTree.models import InvenTreeAttachment
+from InvenTree.models import InvenTreeAttachment, ReferenceIndexingMixin
 
 
 def get_next_po_number():
@@ -89,7 +89,7 @@ def get_next_so_number():
     return reference
 
 
-class Order(models.Model):
+class Order(ReferenceIndexingMixin):
     """ Abstract model for an order.
 
     Instances of this class:
@@ -147,6 +147,9 @@ class Order(models.Model):
         return new_ref
 
     def save(self, *args, **kwargs):
+
+        self.rebuild_reference_field()
+
         if not self.creation_date:
             self.creation_date = datetime.now().date()
 
@@ -531,6 +534,12 @@ class SalesOrder(Order):
 
         return queryset
 
+    def save(self, *args, **kwargs):
+
+        self.rebuild_reference_field()
+
+        super().save(*args, **kwargs)
+
     def __str__(self):
 
         prefix = getSetting('SALESORDER_REFERENCE_PREFIX')

InvenTree/order/test_migrations.py

@@ -0,0 +1,59 @@
+"""
+Unit tests for the 'order' model data migrations
+"""
+
+from django_test_migrations.contrib.unittest_case import MigratorTestCase
+
+from InvenTree import helpers
+
+
+class TestForwardMigrations(MigratorTestCase):
+    """
+    Test entire schema migration
+    """
+
+    migrate_from = ('order', helpers.getOldestMigrationFile('order'))
+    migrate_to = ('order', helpers.getNewestMigrationFile('order'))
+
+    def prepare(self):
+        """
+        Create initial data set
+        """
+
+        # Create a purchase order from a supplier
+        Company = self.old_state.apps.get_model('company', 'company')
+
+        supplier = Company.objects.create(
+            name='Supplier A',
+            description='A great supplier!',
+            is_supplier=True
+        )
+
+        PurchaseOrder = self.old_state.apps.get_model('order', 'purchaseorder')
+
+        # Create some orders
+        for ii in range(10):
+
+            order = PurchaseOrder.objects.create(
+                supplier=supplier,
+                reference=f"{ii}-abcde",
+                description="Just a test order"
+            )
+
+            # Initially, the 'reference_int' field is unavailable
+            with self.assertRaises(AttributeError):
+                print(order.reference_int)
+
+    def test_ref_field(self):
+        """
+        Test that the 'reference_int' field has been created and is filled out correctly
+        """
+
+        PurchaseOrder = self.new_state.apps.get_model('order', 'purchaseorder')
+
+        for ii in range(10):
+
+            order = PurchaseOrder.objects.get(reference=f"{ii}-abcde")
+
+            # The integer reference field must have been correctly updated
+        self.assertEqual(order.reference_int, ii)