Merge pull request #771 from SchrodingersGat/query-speed

Query speed

InvenTree/part/api.py

@@ -6,7 +6,7 @@ Provides a JSON API for the Part app
 from __future__ import unicode_literals
 
 from django_filters.rest_framework import DjangoFilterBackend
-
+from django.http import JsonResponse
 from django.db.models import Q, F, Count
 
 from rest_framework import status
@@ -110,7 +110,7 @@ class PartThumbs(generics.ListAPIView):
 
     serializer_class = part_serializers.PartThumbSerializer
 
-    def list(self, reguest, *args, **kwargs):
+    def list(self, request, *args, **kwargs):
         """
         Serialize the available Part images.
         - Images may be used for multiple parts!
@@ -142,6 +142,7 @@ class PartDetail(generics.RetrieveUpdateAPIView):
 
         queryset = part_serializers.PartSerializer.prefetch_queryset(queryset)
         queryset = part_serializers.PartSerializer.annotate_queryset(queryset)
+
         return queryset
 
     permission_classes = [
@@ -151,15 +152,13 @@ class PartDetail(generics.RetrieveUpdateAPIView):
     def get_serializer(self, *args, **kwargs):
 
         try:
-            cat_detail = str2bool(self.request.query_params.get('category_detail', False))
+            kwargs['category_detail'] = str2bool(self.request.query_params.get('category_detail', False))
         except AttributeError:
-            cat_detail = None
+            pass
 
         # Ensure the request context is passed through
         kwargs['context'] = self.get_serializer_context()
 
-        kwargs['category_detail'] = cat_detail
-
         # Pass a list of "starred" parts fo the current user to the serializer
         # We do this to reduce the number of database queries required!
         if self.starred_parts is None and self.request is not None:
@@ -198,16 +197,9 @@ class PartList(generics.ListCreateAPIView):
 
     def get_serializer(self, *args, **kwargs):
 
-        try:
-            cat_detail = str2bool(self.request.query_params.get('category_detail', False))
-        except AttributeError:
-            cat_detail = None
-
         # Ensure the request context is passed through
         kwargs['context'] = self.get_serializer_context()
 
-        kwargs['category_detail'] = cat_detail
-
         # Pass a list of "starred" parts fo the current user to the serializer
         # We do this to reduce the number of database queries required!
         if self.starred_parts is None and self.request is not None:
@@ -217,6 +209,71 @@ class PartList(generics.ListCreateAPIView):
 
         return self.serializer_class(*args, **kwargs)
 
+    def list(self, request, *args, **kwargs):
+        """
+        Overide the 'list' method, as the PartCategory objects are
+        very expensive to serialize!
+
+        So we will serialize them first, and keep them in memory,
+        so that they do not have to be serialized multiple times...
+        """
+
+        queryset = self.filter_queryset(self.get_queryset())
+
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = self.get_serializer(queryset, many=True)
+
+        data = serializer.data
+
+        # Do we wish to include PartCategory detail?
+        if str2bool(request.query_params.get('category_detail', False)):
+
+            # Work out which part categorie we need to query
+            category_ids = set()
+
+            for part in data:
+                cat_id = part['category']
+
+                if cat_id is not None:
+                    category_ids.add(cat_id)
+
+            # Fetch only the required PartCategory objects from the database
+            categories = PartCategory.objects.filter(pk__in=category_ids).prefetch_related(
+                'parts',
+                'parent',
+                'children',
+            )
+
+            category_map = {}
+
+            # Serialize each PartCategory object
+            for category in categories:
+                category_map[category.pk] = part_serializers.CategorySerializer(category).data
+
+            for part in data:
+                cat_id = part['category']
+
+                if cat_id is not None and cat_id in category_map.keys():
+                    detail = category_map[cat_id]
+                else:
+                    detail = None
+
+                part['category_detail'] = detail
+
+        """
+        Determine the response type based on the request.
+        a) For HTTP requests (e.g. via the browseable API) return a DRF response
+        b) For AJAX requests, simply return a JSON rendered response.
+        """
+        if request.is_ajax():
+            return JsonResponse(data, safe=False)
+        else:
+            return Response(data)
+
     def create(self, request, *args, **kwargs):
         """ Override the default 'create' behaviour:
         We wish to save the user who created this part!
@@ -239,15 +296,16 @@ class PartList(generics.ListCreateAPIView):
 
         queryset = super().get_queryset(*args, **kwargs)
         queryset = part_serializers.PartSerializer.prefetch_queryset(queryset)
+        queryset = part_serializers.PartSerializer.annotate_queryset(queryset)
 
         return queryset
 
     def filter_queryset(self, queryset):
         """
-        Perform custom filtering of the queryset
+        Perform custom filtering of the queryset.
+        We overide the DRF filter_fields here because
         """
 
-        # Perform basic filtering
         queryset = super().filter_queryset(queryset)
 
         # Filter by 'starred' parts?

InvenTree/part/serializers.py

@@ -101,6 +101,8 @@ class PartSerializer(InvenTreeModelSerializer):
 
         return queryset.prefetch_related(
             'category',
+            'category__parts',
+            'category__parent',
             'stock_items',
             'bom_items',
             'builds',
@@ -128,12 +130,7 @@ class PartSerializer(InvenTreeModelSerializer):
 
         # Annotate the number total stock count
         queryset = queryset.annotate(
-            in_stock=Coalesce(Sum('stock_items__quantity', filter=stock_filter, distinct=True), Decimal(0))
+            in_stock=Coalesce(Sum('stock_items__quantity', filter=stock_filter, distinct=True), Decimal(0)),
-        )
-
-        # Annotate the number of parts "on order"
-        # Total "on order" parts = "Quantity" - "Received" for each active purchase order
-        queryset = queryset.annotate(
             ordering=Coalesce(Sum(
                 'supplier_parts__purchase_order_line_items__quantity',
                 filter=order_filter,
@@ -142,11 +139,7 @@ class PartSerializer(InvenTreeModelSerializer):
                 'supplier_parts__purchase_order_line_items__received',
                 filter=order_filter,
                 distinct=True
-            ), Decimal(0))
+            ), Decimal(0)),
-        )
-
-        # Annotate number of parts being build
-        queryset = queryset.annotate(
             building=Coalesce(
                 Sum('builds__quantity', filter=build_filter, distinct=True), Decimal(0)
             )

InvenTree/stock/api.py

@@ -7,15 +7,20 @@ from django_filters import NumberFilter
 
 from django.conf.urls import url, include
 from django.urls import reverse
+from django.http import JsonResponse
 from django.db.models import Q
 
 from .models import StockLocation, StockItem
 from .models import StockItemTracking
 
 from part.models import Part, PartCategory
+from part.serializers import PartBriefSerializer
+
+from company.models import SupplierPart
+from company.serializers import SupplierPartSerializer
 
 from .serializers import StockItemSerializer
-from .serializers import LocationSerializer
+from .serializers import LocationSerializer, LocationBriefSerializer
 from .serializers import StockTrackingSerializer
 
 from InvenTree.views import TreeSerializer
@@ -322,45 +327,153 @@ class StockList(generics.ListCreateAPIView):
     """
 
     serializer_class = StockItemSerializer
-
     queryset = StockItem.objects.all()
 
-    def get_serializer(self, *args, **kwargs):
-
-        try:
-            kwargs['part_detail'] = str2bool(self.request.query_params.get('part_detail', None))
-        except AttributeError:
-            pass
-
-        try:
-            kwargs['location_detail'] = str2bool(self.request.query_params.get('location_detail', None))
-        except AttributeError:
-            pass
-
-        try:
-            kwargs['supplier_part_detail'] = str2bool(self.request.query_params.get('supplier_part_detail', None))
-        except AttributeError:
-            pass
-        
-        # Ensure the request context is passed through
-        kwargs['context'] = self.get_serializer_context()
-
-        return self.serializer_class(*args, **kwargs)
-
     # TODO - Override the 'create' method for this view,
     # to allow the user to be recorded when a new StockItem object is created
 
+    def list(self, request, *args, **kwargs):
+        """
+        Override the 'list' method, as the StockLocation objects
+        are very expensive to serialize.
+        
+        So, we fetch and serialize the required StockLocation objects only as required.
+        """
+
+        queryset = self.filter_queryset(self.get_queryset())
+
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = self.get_serializer(queryset, many=True)
+
+        data = serializer.data
+
+        # Keep track of which related models we need to query
+        location_ids = set()
+        part_ids = set()
+        supplier_part_ids = set()
+
+        # Iterate through each StockItem and grab some data
+        for item in data:
+            loc = item['location']
+            if loc:
+                location_ids.add(loc)
+
+            part = item['part']
+            if part:
+                part_ids.add(part)
+
+            sp = item['supplier_part']
+            if sp:
+                supplier_part_ids.add(sp)
+
+        # Do we wish to include Part detail?
+        if str2bool(request.query_params.get('part_detail', False)):
+
+            # Fetch only the required Part objects from the database
+            parts = Part.objects.filter(pk__in=part_ids).prefetch_related(
+                'category',
+            )
+
+            part_map = {}
+
+            for part in parts:
+                part_map[part.pk] = PartBriefSerializer(part).data
+
+            # Now update each StockItem with the related Part data
+            for stock_item in data:
+                part_id = stock_item['part']
+                stock_item['part_detail'] = part_map.get(part_id, None)
+
+        # Do we wish to include SupplierPart detail?
+        if str2bool(request.query_params.get('supplier_part_detail', False)):
+
+            supplier_parts = SupplierPart.objects.filter(pk__in=supplier_part_ids)
+
+            supplier_part_map = {}
+
+            for part in supplier_parts:
+                supplier_part_map[part.pk] = SupplierPartSerializer(part).data
+
+            for stock_item in data:
+                part_id = stock_item['supplier_part']
+                stock_item['supplier_part_detail'] = supplier_part_map.get(part_id, None)
+
+        # Do we wish to include StockLocation detail?
+        if str2bool(request.query_params.get('location_detail', False)):
+
+            # Fetch only the required StockLocation objects from the database
+            locations = StockLocation.objects.filter(pk__in=location_ids).prefetch_related(
+                'parent',
+                'children',
+            )
+
+            location_map = {}
+
+            # Serialize each StockLocation object
+            for location in locations:
+                location_map[location.pk] = LocationBriefSerializer(location).data
+
+            # Now update each StockItem with the related StockLocation data
+            for stock_item in data:
+                loc_id = stock_item['location']
+                stock_item['location_detail'] = location_map.get(loc_id, None)
+
+        """
+        Determine the response type based on the request.
+        a) For HTTP requests (e.g. via the browseable API) return a DRF response
+        b) For AJAX requests, simply return a JSON rendered response.
+
+        Note: b) is about 100x quicker than a), because the DRF framework adds a lot of cruft
+        """
+
+        if request.is_ajax():
+            return JsonResponse(data, safe=False)
+        else:
+            return Response(data)
+
     def get_queryset(self, *args, **kwargs):
 
         queryset = super().get_queryset(*args, **kwargs)
         queryset = StockItemSerializer.prefetch_queryset(queryset)
+        queryset = StockItemSerializer.annotate_queryset(queryset)
 
         return queryset
 
     def filter_queryset(self, queryset):
 
-        # Start with all objects
+        params = self.request.query_params
-        stock_list = super().filter_queryset(queryset)
+
+        # Perform basic filtering:
+        # Note: We do not let DRF filter here, it be slow AF
+
+        supplier_part = params.get('supplier_part', None)
+
+        if supplier_part:
+            queryset = queryset.filter(supplier_part=supplier_part)
+
+        belongs_to = params.get('belongs_to', None)
+
+        if belongs_to:
+            queryset = queryset.filter(belongs_to=belongs_to)
+
+        build = params.get('build', None)
+
+        if build:
+            queryset = queryset.filter(build=build)
+
+        build_order = params.get('build_order', None)
+
+        if build_order:
+            queryset = queryset.filter(build_order=build_order)
+
+        sales_order = params.get('sales_order', None)
+
+        if sales_order:
+            queryset = queryset.filter(sales_order=sales_order)
         
         in_stock = self.request.query_params.get('in_stock', None)
 
@@ -369,10 +482,10 @@ class StockList(generics.ListCreateAPIView):
 
             if in_stock:
                 # Filter out parts which are not actually "in stock"
-                stock_list = stock_list.filter(StockItem.IN_STOCK_FILTER)
+                queryset = queryset.filter(StockItem.IN_STOCK_FILTER)
             else:
                 # Only show parts which are not in stock
-                stock_list = stock_list.exclude(StockItem.IN_STOCK_FILTER)
+                queryset = queryset.exclude(StockItem.IN_STOCK_FILTER)
 
         # Filter by 'allocated' patrs?
         allocated = self.request.query_params.get('allocated', None)
@@ -382,17 +495,17 @@ class StockList(generics.ListCreateAPIView):
 
             if allocated:
                 # Filter StockItem with either build allocations or sales order allocations
-                stock_list = stock_list.filter(Q(sales_order_allocations__isnull=False) | Q(allocations__isnull=False))
+                queryset = queryset.filter(Q(sales_order_allocations__isnull=False) | Q(allocations__isnull=False))
             else:
                 # Filter StockItem without build allocations or sales order allocations
-                stock_list = stock_list.filter(Q(sales_order_allocations__isnull=True) & Q(allocations__isnull=True))
+                queryset = queryset.filter(Q(sales_order_allocations__isnull=True) & Q(allocations__isnull=True))
                 
         # Do we wish to filter by "active parts"
         active = self.request.query_params.get('active', None)
 
         if active is not None:
             active = str2bool(active)
-            stock_list = stock_list.filter(part__active=active)
+            queryset = queryset.filter(part__active=active)
 
         # Does the client wish to filter by the Part ID?
         part_id = self.request.query_params.get('part', None)
@@ -403,9 +516,9 @@ class StockList(generics.ListCreateAPIView):
 
                 # If the part is a Template part, select stock items for any "variant" parts under that template
                 if part.is_template:
-                    stock_list = stock_list.filter(part__in=[part.id for part in Part.objects.filter(variant_of=part_id)])
+                    queryset = queryset.filter(part__in=[part.id for part in Part.objects.filter(variant_of=part_id)])
                 else:
-                    stock_list = stock_list.filter(part=part_id)
+                    queryset = queryset.filter(part=part_id)
 
             except (ValueError, Part.DoesNotExist):
                 raise ValidationError({"part": "Invalid Part ID specified"})
@@ -418,7 +531,7 @@ class StockList(generics.ListCreateAPIView):
                 ancestor = StockItem.objects.get(pk=anc_id)
 
                 # Only allow items which are descendants of the specified StockItem
-                stock_list = stock_list.filter(id__in=[item.pk for item in ancestor.children.all()])
+                queryset = queryset.filter(id__in=[item.pk for item in ancestor.children.all()])
 
             except (ValueError, Part.DoesNotExist):
                 raise ValidationError({"ancestor": "Invalid ancestor ID specified"})
@@ -432,15 +545,15 @@ class StockList(generics.ListCreateAPIView):
 
             # Filter by 'null' location (i.e. top-level items)
             if isNull(loc_id):
-                stock_list = stock_list.filter(location=None)
+                queryset = queryset.filter(location=None)
             else:
                 try:
                     # If '?cascade=true' then include items which exist in sub-locations
                     if cascade:
                         location = StockLocation.objects.get(pk=loc_id)
-                        stock_list = stock_list.filter(location__in=location.getUniqueChildren())
+                        queryset = queryset.filter(location__in=location.getUniqueChildren())
                     else:
-                        stock_list = stock_list.filter(location=loc_id)
+                        queryset = queryset.filter(location=loc_id)
                     
                 except (ValueError, StockLocation.DoesNotExist):
                     pass
@@ -451,7 +564,7 @@ class StockList(generics.ListCreateAPIView):
         if cat_id:
             try:
                 category = PartCategory.objects.get(pk=cat_id)
-                stock_list = stock_list.filter(part__category__in=category.getUniqueChildren())
+                queryset = queryset.filter(part__category__in=category.getUniqueChildren())
 
             except (ValueError, PartCategory.DoesNotExist):
                 raise ValidationError({"category": "Invalid category id specified"})
@@ -460,44 +573,42 @@ class StockList(generics.ListCreateAPIView):
         status = self.request.query_params.get('status', None)
 
         if status:
-            stock_list = stock_list.filter(status=status)
+            queryset = queryset.filter(status=status)
 
         # Filter by supplier_part ID
         supplier_part_id = self.request.query_params.get('supplier_part', None)
 
         if supplier_part_id:
-            stock_list = stock_list.filter(supplier_part=supplier_part_id)
+            queryset = queryset.filter(supplier_part=supplier_part_id)
 
         # Filter by company (either manufacturer or supplier)
         company = self.request.query_params.get('company', None)
 
         if company is not None:
-            stock_list = stock_list.filter(Q(supplier_part__supplier=company) | Q(supplier_part__manufacturer=company))
+            queryset = queryset.filter(Q(supplier_part__supplier=company) | Q(supplier_part__manufacturer=company))
 
         # Filter by supplier
         supplier = self.request.query_params.get('supplier', None)
 
         if supplier is not None:
-            stock_list = stock_list.filter(supplier_part__supplier=supplier)
+            queryset = queryset.filter(supplier_part__supplier=supplier)
 
         # Filter by manufacturer
         manufacturer = self.request.query_params.get('manufacturer', None)
 
         if manufacturer is not None:
-            stock_list = stock_list.filter(supplier_part__manufacturer=manufacturer)
+            queryset = queryset.filter(supplier_part__manufacturer=manufacturer)
 
         # Also ensure that we pre-fecth all the related items
-        stock_list = stock_list.prefetch_related(
+        queryset = queryset.prefetch_related(
             'part',
             'part__category',
             'location'
         )
 
-        stock_list = stock_list.order_by('part__name')
+        queryset = queryset.order_by('part__name')
 
-        return stock_list
+        return queryset
-
-    serializer_class = StockItemSerializer
 
     permission_classes = [
         permissions.IsAuthenticated,
@@ -510,12 +621,6 @@ class StockList(generics.ListCreateAPIView):
     ]
 
     filter_fields = [
-        'supplier_part',
-        'belongs_to',
-        'build',
-        'build_order',
-        'sales_order',
-        'build_order',
     ]
 
 

InvenTree/stock/serializers.py

@@ -7,6 +7,9 @@ from rest_framework import serializers
 from .models import StockItem, StockLocation
 from .models import StockItemTracking
 
+from django.db.models import Sum, Count
+from django.db.models.functions import Coalesce
+
 from company.serializers import SupplierPartSerializer
 from part.serializers import PartBriefSerializer
 from InvenTree.serializers import UserSerializerBrief, InvenTreeModelSerializer
@@ -17,15 +20,12 @@ class LocationBriefSerializer(InvenTreeModelSerializer):
     Provides a brief serializer for a StockLocation object
     """
 
-    url = serializers.CharField(source='get_absolute_url', read_only=True)
-
     class Meta:
         model = StockLocation
         fields = [
             'pk',
             'name',
             'pathstring',
-            'url',
         ]
 
 
@@ -65,6 +65,10 @@ class StockItemSerializer(InvenTreeModelSerializer):
         """
 
         return queryset.prefetch_related(
+            'belongs_to',
+            'build',
+            'build_order',
+            'sales_order',
             'supplier_part',
             'supplier_part__supplier',
             'supplier_part__manufacturer',
@@ -82,7 +86,13 @@ class StockItemSerializer(InvenTreeModelSerializer):
         performing database queries as efficiently as possible.
         """
 
-        # TODO - Add custom annotated fields
+        queryset = queryset.annotate(
+            allocated=Coalesce(
+                Sum('sales_order_allocations__quantity', distinct=True), 0) + Coalesce(
+                Sum('allocations__quantity', distinct=True), 0),
+            tracking_items=Count('tracking_info'),
+        )
+
         return queryset
 
     status_text = serializers.CharField(source='get_status_display', read_only=True)
@@ -91,10 +101,10 @@ class StockItemSerializer(InvenTreeModelSerializer):
     location_detail = LocationBriefSerializer(source='location', many=False, read_only=True)
     supplier_part_detail = SupplierPartSerializer(source='supplier_part', many=False, read_only=True)
 
-    tracking_items = serializers.IntegerField(source='tracking_info_count', read_only=True)
+    tracking_items = serializers.IntegerField()
 
     quantity = serializers.FloatField()
-    allocated = serializers.FloatField(source='allocation_count', read_only=True)
+    allocated = serializers.FloatField()
 
     def __init__(self, *args, **kwargs):
 
@@ -143,6 +153,7 @@ class StockItemSerializer(InvenTreeModelSerializer):
         They can be updated by accessing the appropriate API endpoints
         """
         read_only_fields = [
+            'allocated',
             'stocktake_date',
             'stocktake_user',
             'updated',

InvenTree/stock/templates/stock/item_base.html

@@ -105,9 +105,8 @@ InvenTree | {% trans "Stock Item" %} - {{ item }}
     <col width='25'>
     <tr>
         <td><span class='fas fa-shapes'></span></td>
-        <td>Part</td>
+        <td>{% trans "Base Part" %}</td>
         <td>
-            {% include "hover_image.html" with image=item.part.image hover=True %}
             <a href="{% url 'part-stock' item.part.id %}">{{ item.part.full_name }}
         </td>
     </tr>
@@ -145,7 +144,7 @@ InvenTree | {% trans "Stock Item" %} - {{ item }}
     {% endif %}
     {% if item.serialized %}
     <tr>
-        <td></td>
+        <td><span class='fas fa-hashtag'></span></td>
         <td>{% trans "Serial Number" %}</td>
         <td>{{ item.serial }}</td>
     </tr>