Fixes for aggregation issues

- Ensure that "distinct=True" is set!
- ARRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH

InvenTree/part/api.py

@@ -166,6 +166,8 @@ class PartList(generics.ListCreateAPIView):
 
     serializer_class = part_serializers.PartSerializer
 
+    queryset = Part.objects.all()
+
     def create(self, request, *args, **kwargs):
         """ Override the default 'create' behaviour:
         We wish to save the user who created this part!
@@ -184,7 +186,89 @@ class PartList(generics.ListCreateAPIView):
         headers = self.get_success_headers(serializer.data)
         return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
 
-    def list(self, request, *args, **kwargs):
+
+    def filter_queryset(self, queryset):
+        """
+        Perform custom filtering of the queryset
+        """
+
+        # Perform basic filtering
+        queryset = super().filter_queryset(queryset)
+
+        # Filter by 'starred' parts?
+        starred = str2bool(self.request.query_params.get('starred', None))
+
+        if starred is not None:
+            starred_parts = [star.part.pk for star in self.request.user.starred_parts.all()]
+
+            if starred:
+                queryset = queryset.filter(pk__in=starred_parts)
+            else:
+                queryset = queryset.exclude(pk__in=starred_parts)
+
+        # Cascade?
+        cascade = str2bool(self.request.query_params.get('cascade', None))
+
+        # Does the user wish to filter by category?
+        cat_id = self.request.query_params.get('category', None)
+
+        if cat_id is None:
+            # No category filtering if category is not specified
+            pass
+        
+        else:
+            # Category has been specified!
+            if isNull(cat_id):
+                # A 'null' category is the top-level category
+                if cascade is False:
+                    # Do not cascade, only list parts in the top-level category
+                    queryset = queryset.filter(category=None)
+
+            else:
+                try:
+                    category = PartCategory.objects.get(pk=cat_id)
+
+                    # If '?cascade=true' then include parts which exist in sub-categories
+                    if cascade:
+                        queryset = queryset.filter(category__in=category.getUniqueChildren())
+                    # Just return parts directly in the requested category
+                    else:
+                        queryset = queryset.filter(category=cat_id)
+                except (ValueError, PartCategory.DoesNotExist):
+                    pass
+
+        # Annotate calculated data to the queryset
+        # (This will be used for further filtering)
+        queryset = part_serializers.PartSerializer.annotate_queryset(queryset)
+
+        # Filter by whether the part has stock
+        has_stock = self.request.query_params.get("has_stock", None)
+        if has_stock is not None:
+            has_stock = str2bool(has_stock)
+
+            if has_stock:
+                queryset = queryset.filter(Q(in_stock__gt=0))
+            else:
+                queryset = queryset.filter(Q(in_stock_lte=0))
+
+        # If we are filtering by 'low_stock' status
+        low_stock = self.request.query_params.get('low_stock', None)
+
+        if low_stock is not None:
+            low_stock = str2bool(low_stock)
+
+            if low_stock:
+                # Ignore any parts which do not have a specified 'minimum_stock' level
+                queryset = queryset.exclude(minimum_stock=0)
+                # Filter items which have an 'in_stock' level lower than 'minimum_stock'
+                queryset = queryset.filter(Q(in_stock__lt=F('minimum_stock')))
+            else:
+                # Filter items which have an 'in_stock' level higher than 'minimum_stock'
+                queryset = queryset.filter(Q(in_stock__gte=F('minimum_stock')))
+
+        return queryset
+
+    def dolist(self, request, *args, **kwargs):
         """
         Instead of using the DRF serialiser to LIST,
         we serialize the objects manually.
@@ -193,10 +277,11 @@ class PartList(generics.ListCreateAPIView):
 
         queryset = self.filter_queryset(self.get_queryset())
 
-        # Filters for annotations
+        queryset = part_serializers.PartSerializer.prefetch_queryset(queryset)
 
-        # "in_stock" count should only sum stock items which are "in stock"
+        queryset = part_serializers.PartSerializer.annotate_queryset(queryset)
-        stock_filter = Q(stock_items__status__in=StockStatus.AVAILABLE_CODES)
+
+        # Filters for annotations
 
         # "on_order" items should only sum orders which are currently outstanding
         order_filter = Q(supplier_parts__purchase_order_line_items__order__status__in=OrderStatus.OPEN)
@@ -225,9 +310,9 @@ class PartList(generics.ListCreateAPIView):
             'active',
         ).annotate(
             # Quantity of items which are "in stock"
-            in_stock=Coalesce(Sum('stock_items__quantity', filter=stock_filter), Decimal(0)),
+            in_stock=Coalesce(Sum('stock_items__quantity'), Decimal(0)) #, filter=stock_filter), Decimal(0)),
-            on_order=Coalesce(Sum('supplier_parts__purchase_order_line_items__quantity', filter=order_filter), Decimal(0)),
+            #on_order=Coalesce(Sum('supplier_parts__purchase_order_line_items__quantity', filter=order_filter), Decimal(0)),
-            building=Coalesce(Sum('builds__quantity', filter=build_filter), Decimal(0)),
+            #building=Coalesce(Sum('builds__quantity', filter=build_filter), Decimal(0)),
         )
 
         # If we are filtering by 'has_stock' status
@@ -300,60 +385,6 @@ class PartList(generics.ListCreateAPIView):
 
         return Response(data)
 
-    def get_queryset(self):
-        """
-        Implement custom filtering for the Part list API
-        """
-
-        # Start with all objects
-        parts_list = Part.objects.all()
-
-        # Filter by 'starred' parts?
-        starred = str2bool(self.request.query_params.get('starred', None))
-
-        if starred is not None:
-            starred_parts = [star.part.pk for star in self.request.user.starred_parts.all()]
-
-            if starred:
-                parts_list = parts_list.filter(pk__in=starred_parts)
-            else:
-                parts_list = parts_list.exclude(pk__in=starred_parts)
-
-        cascade = str2bool(self.request.query_params.get('cascade', None))
-        
-        # Does the user wish to filter by category?
-        cat_id = self.request.query_params.get('category', None)
-
-        if cat_id is None:
-            # No category filtering if category is not specified
-            pass
-        
-        else:
-            # Category has been specified!
-            if isNull(cat_id):
-                # A 'null' category is the top-level category
-                if cascade is False:
-                    # Do not cascade, only list parts in the top-level category
-                    parts_list = parts_list.filter(category=None)
-
-            else:
-                try:
-                    category = PartCategory.objects.get(pk=cat_id)
-
-                    # If '?cascade=true' then include parts which exist in sub-categories
-                    if cascade:
-                        parts_list = parts_list.filter(category__in=category.getUniqueChildren())
-                    # Just return parts directly in the requested category
-                    else:
-                        parts_list = parts_list.filter(category=cat_id)
-                except (ValueError, PartCategory.DoesNotExist):
-                    pass
-
-        # Ensure that related models are pre-loaded to reduce DB trips
-        parts_list = self.get_serializer_class().setup_eager_loading(parts_list)
-
-        return parts_list
-
     permission_classes = [
         permissions.IsAuthenticated,
     ]

InvenTree/part/serializers.py

@@ -10,6 +10,12 @@ from .models import PartCategory
 from .models import BomItem
 from .models import PartParameter, PartParameterTemplate
 
+from decimal import Decimal
+
+from django.db.models import Q, F, Sum, Count
+from django.db.models.functions import Coalesce
+
+from InvenTree.status_codes import StockStatus, OrderStatus, BuildStatus
 from InvenTree.serializers import InvenTreeModelSerializer
 
 
@@ -78,24 +84,77 @@ class PartSerializer(InvenTreeModelSerializer):
     Used when displaying all details of a single component.
     """
 
-    allocated_stock = serializers.FloatField(source='allocation_count', read_only=True)
+    @staticmethod
-    bom_items = serializers.IntegerField(source='bom_count', read_only=True)
+    def prefetch_queryset(queryset):
-    building = serializers.FloatField(source='quantity_being_built', read_only=False)
+        return queryset.prefetch_related(
-    category_name = serializers.CharField(source='category_path', read_only=True)
+            'category',
-    image = serializers.CharField(source='get_image_url', read_only=True)
+            'stock_items',
-    on_order = serializers.FloatField(read_only=True)
+            'bom_items',
-    thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True)
+            'builds',
-    url = serializers.CharField(source='get_absolute_url', read_only=True)
+            'supplier_parts',
-    used_in = serializers.IntegerField(source='used_in_count', read_only=True)
+            'supplier_parts__purchase_order_line_items',
+            'supplier_parts__purcahes_order_line_items__order'
+        )
 
     @staticmethod
-    def setup_eager_loading(queryset):
+    def annotate_queryset(queryset):
-        queryset = queryset.prefetch_related('category')
+        """
-        queryset = queryset.prefetch_related('stock_items')
+        Add some extra annotations to the queryset, 
-        queryset = queryset.prefetch_related('bom_items')
+        performing database queries as efficiently as possible,
-        queryset = queryset.prefetch_related('builds')
+        to reduce database trips.
+        """
+
+        # Filter to limit stock items to "available"
+        stock_filter = Q(stock_items__status__in=StockStatus.AVAILABLE_CODES)
+
+        # Filter to limit orders to "open"
+        order_filter = Q(supplier_parts__purchase_order_line_items__order__status__in=OrderStatus.OPEN)
+
+        # Filter to limit builds to "active"
+        build_filter = Q(builds__status__in=BuildStatus.ACTIVE_CODES)
+
+        # Annotate the number total stock count
+        queryset = queryset.annotate(
+            in_stock=Coalesce(Sum('stock_items__quantity', filter=stock_filter, distinct=True), Decimal(0))
+        )
+
+        # Annotate the number of parts "on order"
+        # Total "on order" parts = "Quantity" - "Received" for each active purchase order
+        queryset = queryset.annotate(
+            ordering=Coalesce(Sum(
+                'supplier_parts__purchase_order_line_items__quantity',
+                filter=order_filter,
+                distinct=True
+            ), Decimal(0)) - Coalesce(Sum(
+                'supplier_parts__purchase_order_line_items__received',
+                filter=order_filter,
+                distinct=True
+            ), Decimal(0))
+        )
+
+        # Annotate number of parts being build
+        queryset = queryset.annotate(
+            building=Coalesce(
+                Sum('builds__quantity', filter=build_filter, distinct=True), Decimal(0)
+            )
+        )
+        
         return queryset
 
+    in_stock = serializers.FloatField(read_only=True)
+    ordering = serializers.FloatField(read_only=True)
+    building = serializers.FloatField(read_only=True)
+
+    #allocated_stock = serializers.FloatField(source='allocation_count', read_only=True)
+    #bom_items = serializers.IntegerField(source='bom_count', read_only=True)
+    #building = serializers.FloatField(source='quantity_being_built', read_only=False)
+    #category_name = serializers.CharField(source='category_path', read_only=True)
+    image = serializers.CharField(source='get_image_url', read_only=True)
+    #on_order = serializers.FloatField(read_only=True)
+    thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True)
+    url = serializers.CharField(source='get_absolute_url', read_only=True)
+    #used_in = serializers.IntegerField(source='used_in_count', read_only=True)
+
     # TODO - Include a 'category_detail' field which serializers the category object
 
     class Meta:
@@ -103,31 +162,34 @@ class PartSerializer(InvenTreeModelSerializer):
         partial = True
         fields = [
             'active',
-            'allocated_stock',
+            #'allocated_stock',
             'assembly',
-            'bom_items',
+            #'bom_items',
-            'building',
+            #'building',
             'category',
-            'category_name',
+            #'category_name',
             'component',
             'description',
             'full_name',
             'image',
+            'in_stock',
+            'ordering',
+            'building',
             'IPN',
             'is_template',
             'keywords',
             'link',
             'name',
             'notes',
-            'on_order',
+            #'on_order',
             'pk',
             'purchaseable',
             'salable',
             'thumbnail',
             'trackable',
-            'total_stock',
+            #'total_stock',
             'units',
-            'used_in',
+            #'used_in',
             'url',  # Link to the part detail page
             'variant_of',
             'virtual',