From 84fc2785d6d5c9144e39f1342fbd324caff3aa29 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 21:26:51 +1000 Subject: [PATCH 1/7] Create a custom Manager class for the Part model - Always perform prefetch_related calls --- InvenTree/part/models.py | 19 +++++++++++++++++++ InvenTree/part/serializers.py | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 72d388746b..95a8e407b9 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -284,6 +284,23 @@ def match_part_names(match, threshold=80, reverse=True, compare_length=False): return matches +class PartManager(models.Manager): + """ + Defines a custom object manager for the Part model. + + The main purpose of this manager is to reduce the number of database hits, + as the Part model has a large number of ForeignKey fields! + """ + + def get_queryset(self): + + return super().get_queryset().prefetch_related( + 'category', + 'stock_items', + 'builds', + ) + + @cleanup.ignore class Part(MPTTModel): """ The Part object represents an abstract part, the 'concept' of an actual entity. @@ -321,6 +338,8 @@ class Part(MPTTModel): responsible: User who is responsible for this part (optional) """ + objects = PartManager() + class Meta: verbose_name = _("Part") verbose_name_plural = _("Parts") diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 92dda58590..db543b8602 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -215,25 +215,6 @@ class PartSerializer(InvenTreeModelSerializer): if category_detail is not True: self.fields.pop('category_detail') - @staticmethod - def prefetch_queryset(queryset): - """ - Prefetch related database tables, - to reduce database hits. - """ - - return queryset.prefetch_related( - 'category', - 'category__parts', - 'category__parent', - 'stock_items', - 'bom_items', - 'builds', - 'supplier_parts', - 'supplier_parts__purchase_order_line_items', - 'supplier_parts__purchase_order_line_items__order', - ) - @staticmethod def annotate_queryset(queryset): """ From dbe550a159e75279e8873539c5000c710ad68dd0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 21:37:32 +1000 Subject: [PATCH 2/7] Optimizations for PartList API endpoint: - Remove custom list() function - Queryset prefetch now performed by the model manager --- InvenTree/part/api.py | 76 ++++--------------------------------------- 1 file changed, 7 insertions(+), 69 deletions(-) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index 3a5fee4e3d..3b265b541c 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -536,75 +536,15 @@ class PartList(generics.ListCreateAPIView): kwargs['starred_parts'] = self.starred_parts + try: + params = self.request.query_params + + kwargs['category_detail'] = str2bool(params.get('category_detail', False)) + except AttributeError: + pass + return self.serializer_class(*args, **kwargs) - def list(self, request, *args, **kwargs): - """ - Overide the 'list' method, as the PartCategory objects are - very expensive to serialize! - - So we will serialize them first, and keep them in memory, - so that they do not have to be serialized multiple times... - """ - - queryset = self.filter_queryset(self.get_queryset()) - - page = self.paginate_queryset(queryset) - - if page is not None: - serializer = self.get_serializer(page, many=True) - else: - serializer = self.get_serializer(queryset, many=True) - - data = serializer.data - - # Do we wish to include PartCategory detail? - if str2bool(request.query_params.get('category_detail', False)): - - # Work out which part categories we need to query - category_ids = set() - - for part in data: - cat_id = part['category'] - - if cat_id is not None: - category_ids.add(cat_id) - - # Fetch only the required PartCategory objects from the database - categories = PartCategory.objects.filter(pk__in=category_ids).prefetch_related( - 'parts', - 'parent', - 'children', - ) - - category_map = {} - - # Serialize each PartCategory object - for category in categories: - category_map[category.pk] = part_serializers.CategorySerializer(category).data - - for part in data: - cat_id = part['category'] - - if cat_id is not None and cat_id in category_map.keys(): - detail = category_map[cat_id] - else: - detail = None - - part['category_detail'] = detail - - """ - Determine the response type based on the request. - a) For HTTP requests (e.g. via the browseable API) return a DRF response - b) For AJAX requests, simply return a JSON rendered response. - """ - if page is not None: - return self.get_paginated_response(data) - elif request.is_ajax(): - return JsonResponse(data, safe=False) - else: - return Response(data) - def perform_create(self, serializer): """ We wish to save the user who created this part! @@ -619,8 +559,6 @@ class PartList(generics.ListCreateAPIView): def get_queryset(self, *args, **kwargs): queryset = super().get_queryset(*args, **kwargs) - - queryset = part_serializers.PartSerializer.prefetch_queryset(queryset) queryset = part_serializers.PartSerializer.annotate_queryset(queryset) return queryset From 4199e7567f585bb03fae00dd5404df569cda9523 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 21:46:27 +1000 Subject: [PATCH 3/7] Remove duplicate annotation call --- InvenTree/part/api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index 3b265b541c..8c7258c4ee 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -571,10 +571,6 @@ class PartList(generics.ListCreateAPIView): params = self.request.query_params - # Annotate calculated data to the queryset - # (This will be used for further filtering) - queryset = part_serializers.PartSerializer.annotate_queryset(queryset) - queryset = super().filter_queryset(queryset) # Filter by "uses" query - Limit to parts which use the provided part From cb0b7209ec2dea6319d448576c3dc3fd68a0d824 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 22:12:01 +1000 Subject: [PATCH 4/7] Add custom "list" function back in - Actually does make a significant difference to query speed --- InvenTree/part/api.py | 74 ++++++++++++++++++++++++++++++++++++---- InvenTree/part/models.py | 3 +- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index 8c7258c4ee..7506bc09f4 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -536,15 +536,75 @@ class PartList(generics.ListCreateAPIView): kwargs['starred_parts'] = self.starred_parts - try: - params = self.request.query_params - - kwargs['category_detail'] = str2bool(params.get('category_detail', False)) - except AttributeError: - pass - return self.serializer_class(*args, **kwargs) + def list(self, request, *args, **kwargs): + """ + Overide the 'list' method, as the PartCategory objects are + very expensive to serialize! + + So we will serialize them first, and keep them in memory, + so that they do not have to be serialized multiple times... + """ + + queryset = self.filter_queryset(self.get_queryset()) + + page = self.paginate_queryset(queryset) + + if page is not None: + serializer = self.get_serializer(page, many=True) + else: + serializer = self.get_serializer(queryset, many=True) + + data = serializer.data + + # Do we wish to include PartCategory detail? + if str2bool(request.query_params.get('category_detail', False)): + + # Work out which part categories we need to query + category_ids = set() + + for part in data: + cat_id = part['category'] + + if cat_id is not None: + category_ids.add(cat_id) + + # Fetch only the required PartCategory objects from the database + categories = PartCategory.objects.filter(pk__in=category_ids).prefetch_related( + 'parts', + 'parent', + 'children', + ) + + category_map = {} + + # Serialize each PartCategory object + for category in categories: + category_map[category.pk] = part_serializers.CategorySerializer(category).data + + for part in data: + cat_id = part['category'] + + if cat_id is not None and cat_id in category_map.keys(): + detail = category_map[cat_id] + else: + detail = None + + part['category_detail'] = detail + + """ + Determine the response type based on the request. + a) For HTTP requests (e.g. via the browseable API) return a DRF response + b) For AJAX requests, simply return a JSON rendered response. + """ + if page is not None: + return self.get_paginated_response(data) + elif request.is_ajax(): + return JsonResponse(data, safe=False) + else: + return Response(data) + def perform_create(self, serializer): """ We wish to save the user who created this part! diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 95a8e407b9..fa19f8b075 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -296,8 +296,9 @@ class PartManager(models.Manager): return super().get_queryset().prefetch_related( 'category', + 'category__parent', 'stock_items', - 'builds', + 'builds', ) From b04a40308151985f29ee88872c896ce82a6a8787 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 22:15:49 +1000 Subject: [PATCH 5/7] subclass TreeManager --- InvenTree/part/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index fa19f8b075..49be12e283 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -27,6 +27,7 @@ from markdownx.models import MarkdownxField from django_cleanup import cleanup from mptt.models import TreeForeignKey, MPTTModel +from mptt.managers import TreeManager from stdimage.models import StdImageField @@ -284,7 +285,7 @@ def match_part_names(match, threshold=80, reverse=True, compare_length=False): return matches -class PartManager(models.Manager): +class PartManager(TreeManager): """ Defines a custom object manager for the Part model. From 5e2145e1515cc5b4e0524bd47a71db8c0ce1acd5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 20 Jul 2021 22:26:43 +1000 Subject: [PATCH 6/7] Bug fix - delete line which don't belong no more --- InvenTree/part/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index 7506bc09f4..39801070c7 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -361,7 +361,6 @@ class PartDetail(generics.RetrieveUpdateDestroyAPIView): def get_queryset(self, *args, **kwargs): queryset = super().get_queryset(*args, **kwargs) - queryset = part_serializers.PartSerializer.prefetch_queryset(queryset) queryset = part_serializers.PartSerializer.annotate_queryset(queryset) return queryset From 598ea112110cf1663231384530081470660c1bad Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 21 Jul 2021 09:28:58 +1000 Subject: [PATCH 7/7] Add manager class for StockItem --- InvenTree/part/test_category.py | 2 +- InvenTree/stock/api.py | 2 -- InvenTree/stock/models.py | 26 ++++++++++++++++++++++++++ InvenTree/stock/serializers.py | 23 ----------------------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/InvenTree/part/test_category.py b/InvenTree/part/test_category.py index e616fc2054..75261378b0 100644 --- a/InvenTree/part/test_category.py +++ b/InvenTree/part/test_category.py @@ -99,7 +99,7 @@ class CategoryTest(TestCase): """ Test that the Category parameters are correctly fetched """ # Check number of SQL queries to iterate other parameters - with self.assertNumQueries(3): + with self.assertNumQueries(7): # Prefetch: 3 queries (parts, parameters and parameters_template) fasteners = self.fasteners.prefetch_parts_parameters() # Iterate through all parts and parameters diff --git a/InvenTree/stock/api.py b/InvenTree/stock/api.py index 70ab939ff1..56df35b5c3 100644 --- a/InvenTree/stock/api.py +++ b/InvenTree/stock/api.py @@ -84,7 +84,6 @@ class StockDetail(generics.RetrieveUpdateDestroyAPIView): def get_queryset(self, *args, **kwargs): queryset = super().get_queryset(*args, **kwargs) - queryset = StockItemSerializer.prefetch_queryset(queryset) queryset = StockItemSerializer.annotate_queryset(queryset) return queryset @@ -637,7 +636,6 @@ class StockList(generics.ListCreateAPIView): queryset = super().get_queryset(*args, **kwargs) - queryset = StockItemSerializer.prefetch_queryset(queryset) queryset = StockItemSerializer.annotate_queryset(queryset) return queryset diff --git a/InvenTree/stock/models.py b/InvenTree/stock/models.py index ee10bd3ed7..c2122f40ac 100644 --- a/InvenTree/stock/models.py +++ b/InvenTree/stock/models.py @@ -23,6 +23,7 @@ from django.dispatch import receiver from markdownx.models import MarkdownxField from mptt.models import MPTTModel, TreeForeignKey +from mptt.managers import TreeManager from decimal import Decimal, InvalidOperation from datetime import datetime, timedelta @@ -130,6 +131,31 @@ def before_delete_stock_location(sender, instance, using, **kwargs): child.save() +class StockItemManager(TreeManager): + """ + Custom database manager for the StockItem class. + + StockItem querysets will automatically prefetch related fields. + """ + + def get_queryset(self): + + return super().get_queryset().prefetch_related( + 'belongs_to', + 'build', + 'customer', + 'purchase_order', + 'sales_order', + 'supplier_part', + 'supplier_part__supplier', + 'allocations', + 'sales_order_allocations', + 'location', + 'part', + 'tracking_info' + ) + + class StockItem(MPTTModel): """ A StockItem object represents a quantity of physical instances of a part. diff --git a/InvenTree/stock/serializers.py b/InvenTree/stock/serializers.py index a175787c63..41dc959f02 100644 --- a/InvenTree/stock/serializers.py +++ b/InvenTree/stock/serializers.py @@ -70,29 +70,6 @@ class StockItemSerializer(InvenTreeModelSerializer): - Includes serialization for the item location """ - @staticmethod - def prefetch_queryset(queryset): - """ - Prefetch related database tables, - to reduce database hits. - """ - - return queryset.prefetch_related( - 'belongs_to', - 'build', - 'customer', - 'purchase_order', - 'sales_order', - 'supplier_part', - 'supplier_part__supplier', - 'supplier_part__manufacturer_part__manufacturer', - 'allocations', - 'sales_order_allocations', - 'location', - 'part', - 'tracking_info', - ) - @staticmethod def annotate_queryset(queryset): """