From 73041a3fe657668695aa001148e7b1beffd17302 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 07:53:23 +1000 Subject: [PATCH 01/20] Fix div-by-zero for part pricing --- InvenTree/part/views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/InvenTree/part/views.py b/InvenTree/part/views.py index d5824b2dab..98a209b714 100644 --- a/InvenTree/part/views.py +++ b/InvenTree/part/views.py @@ -567,6 +567,14 @@ class PartPricing(AjaxView): def get_pricing(self, quantity=1): + try: + quantity = int(quantity) + except ValueError: + quantity = 1 + + if quantity < 1: + quantity = 1 + part = self.get_part() ctx = { From 6766063c8b40bc31471a031755a40758798e6abf Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 08:13:22 +1000 Subject: [PATCH 02/20] Reducing request time for PartList API - Initial query time = 5.91s - Prefetch related part categories = 3.64s --- InvenTree/part/api.py | 3 +++ InvenTree/part/serializers.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index aa065e9720..af0e458f47 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -110,6 +110,9 @@ class PartList(generics.ListCreateAPIView): except PartCategory.DoesNotExist: pass + # Ensure that related models are pre-loaded to reduce DB trips + parts_list = self.get_serializer_class().setup_eager_loading(parts_list) + return parts_list permission_classes = [ diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 593f7003f9..0e0c708156 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -58,6 +58,11 @@ class PartSerializer(serializers.ModelSerializer): image_url = serializers.CharField(source='get_image_url', read_only=True) category_name = serializers.CharField(source='category_path', read_only=True) + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('category') + return queryset + class Meta: model = Part partial = True From cee1062b515e0b7ab3f45427d2f72a4232391b55 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 08:16:28 +1000 Subject: [PATCH 03/20] Prefetch locations - 2.6s However the 'total_stock' and 'available_stock' fields are still the majority of the response time --- InvenTree/part/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 0e0c708156..a303ea7aa1 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -61,6 +61,7 @@ class PartSerializer(serializers.ModelSerializer): @staticmethod def setup_eager_loading(queryset): queryset = queryset.prefetch_related('category') + queryset = queryset.prefetch_related('locations') return queryset class Meta: From 50552264903e085dad7fab1710177016721f425a Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 08:31:03 +1000 Subject: [PATCH 04/20] Reduce BOM listing time to ~1.8s --- InvenTree/part/api.py | 8 ++++++-- InvenTree/part/serializers.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index af0e458f47..f3da5ccf40 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -203,10 +203,14 @@ class BomList(generics.ListCreateAPIView): - GET: Return list of BomItem objects - POST: Create a new BomItem object """ - - queryset = BomItem.objects.all() + serializer_class = BomItemSerializer + def get_queryset(self): + queryset = BomItem.objects.all() + queryset = self.get_serializer_class().setup_eager_loading(queryset) + return queryset + permission_classes = [ permissions.IsAuthenticatedOrReadOnly, ] diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index a303ea7aa1..9251580bcc 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -112,16 +112,20 @@ class PartStarSerializer(InvenTreeModelSerializer): class BomItemSerializer(InvenTreeModelSerializer): """ Serializer for BomItem object """ - part_detail = PartBriefSerializer(source='part', many=False, read_only=True) sub_part_detail = PartBriefSerializer(source='sub_part', many=False, read_only=True) price_info = serializers.CharField(read_only=True) + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('sub_part') + queryset = queryset.prefetch_related('sub_part__category') + return queryset + class Meta: model = BomItem fields = [ 'pk', 'part', - 'part_detail', 'sub_part', 'sub_part_detail', 'quantity', From fc75ab7420e0b0c16d08e478bcc0ab8fafeb6741 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 08:44:52 +1000 Subject: [PATCH 05/20] Reduce response time for stock list Initial time - 6.2s After prefetch - 4.3s --- InvenTree/part/serializers.py | 1 + InvenTree/stock/api.py | 3 +++ InvenTree/stock/serializers.py | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 9251580bcc..d4eb82d0a1 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -119,6 +119,7 @@ class BomItemSerializer(InvenTreeModelSerializer): def setup_eager_loading(queryset): queryset = queryset.prefetch_related('sub_part') queryset = queryset.prefetch_related('sub_part__category') + queryset = queryset.prefetch_related('sub_part__locations') return queryset class Meta: diff --git a/InvenTree/stock/api.py b/InvenTree/stock/api.py index 3c08ff8822..2659d3702d 100644 --- a/InvenTree/stock/api.py +++ b/InvenTree/stock/api.py @@ -282,6 +282,9 @@ class StockList(generics.ListCreateAPIView): if supplier_id: stock_list = stock_list.filter(supplier_part__supplier=supplier_id) + # Pre-fetch related objects for better response time + stock_list = self.get_serializer_class().setup_eager_loading(stock_list) + return stock_list serializer_class = StockItemSerializer diff --git a/InvenTree/stock/serializers.py b/InvenTree/stock/serializers.py index a0461ff409..46a1c44845 100644 --- a/InvenTree/stock/serializers.py +++ b/InvenTree/stock/serializers.py @@ -60,6 +60,15 @@ class StockItemSerializer(serializers.ModelSerializer): location = LocationBriefSerializer(many=False, read_only=True) status_text = serializers.CharField(source='get_status_display', read_only=True) + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('part') + queryset = queryset.prefetch_related('part__locations') + queryset = queryset.prefetch_related('part__category') + queryset = queryset.prefetch_related('location') + + return queryset + class Meta: model = StockItem fields = [ From 34620b22b02de7936ae41c18f6155c8516095fdc Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 18:51:57 +1000 Subject: [PATCH 06/20] Add a middleware to count queries - https://www.dabapps.com/blog/logging-sql-queries-django-13/ --- InvenTree/InvenTree/middleware.py | 41 ++++++++++++++++++++ InvenTree/InvenTree/settings.py | 3 ++ InvenTree/part/templates/part/part_base.html | 19 --------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index 3a5c4f059f..c31a6e2b6d 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -1,5 +1,9 @@ from django.shortcuts import HttpResponseRedirect from django.urls import reverse_lazy +from django.db import connection +import logging + +logger = logging.getLogger(__name__) class AuthRequiredMiddleware(object): @@ -24,3 +28,40 @@ class AuthRequiredMiddleware(object): # the view is called. return response + + +class QueryCountMiddleware(object): + """ + This middleware will log the number of queries run + and the total time taken for each request (with a + status code of 200). It does not currently support + multi-db setups. + + Reference: https://www.dabapps.com/blog/logging-sql-queries-django-13/ + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + + response = self.get_response(request) + + if response.status_code == 200: + total_time = 0 + + if len(connection.queries) > 0: + + for query in connection.queries: + query_time = query.get('time') + if query_time is None: + # django-debug-toolbar monkeypatches the connection + # cursor wrapper and adds extra information in each + # item in connection.queries. The query time is stored + # under the key "duration" rather than "time" and is + # in milliseconds, not seconds. + query_time = query.get('duration', 0) / 1000 + total_time += float(query_time) + + logger.debug('%s queries run, total %s seconds' % (len(connection.queries), total_time)) + return response diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 164050d5ca..c6b8e88131 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -86,6 +86,9 @@ MIDDLEWARE = [ 'InvenTree.middleware.AuthRequiredMiddleware' ] +if DEBUG: + MIDDLEWARE.append('InvenTree.middleware.QueryCountMiddleware') + ROOT_URLCONF = 'InvenTree.urls' TEMPLATES = [ diff --git a/InvenTree/part/templates/part/part_base.html b/InvenTree/part/templates/part/part_base.html index 5e4f85d238..8470677487 100644 --- a/InvenTree/part/templates/part/part_base.html +++ b/InvenTree/part/templates/part/part_base.html @@ -85,25 +85,6 @@ {{ part.allocation_count }} {% endif %} - {% if part.supplier_count > 0 %} - - - Price - - - {% if part.min_single_price %} - {% if part.min_single_price == part.max_single_price %} - {{ part.min_single_price }} - {% else %} - {{ part.min_single_price }} to {{ part.max_single_price }} - {% endif %} - from {{ part.supplier_count }} suppliers. - {% else %} - No pricing data avilable - {% endif %} - - - {% endif %} From 70be052c746268b0a13d18a357d15ab47618fc18 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 21:45:37 +1000 Subject: [PATCH 07/20] Rebuilt migrations again (arrrrrrrrrrrrrrrrrrrrrrrrrrgh never do this) --- InvenTree/build/migrations/0001_initial.py | 13 +----- .../migrations/0002_auto_20190520_1922.py | 44 +++++++++++++++++++ InvenTree/company/migrations/0001_initial.py | 2 +- ...519_0004.py => 0002_auto_20190520_1922.py} | 4 +- InvenTree/part/migrations/0001_initial.py | 2 +- ...519_0004.py => 0002_auto_20190520_1922.py} | 6 +-- InvenTree/stock/migrations/0001_initial.py | 8 ++-- InvenTree/stock/models.py | 2 +- 8 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 InvenTree/build/migrations/0002_auto_20190520_1922.py rename InvenTree/company/migrations/{0002_auto_20190519_0004.py => 0002_auto_20190520_1922.py} (96%) rename InvenTree/part/migrations/{0002_auto_20190519_0004.py => 0002_auto_20190520_1922.py} (97%) diff --git a/InvenTree/build/migrations/0001_initial.py b/InvenTree/build/migrations/0001_initial.py index aa7356389a..85fcb1b827 100644 --- a/InvenTree/build/migrations/0001_initial.py +++ b/InvenTree/build/migrations/0001_initial.py @@ -1,6 +1,5 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 -from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion @@ -11,9 +10,6 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('part', '0002_auto_20190519_0004'), - ('stock', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -29,9 +25,6 @@ class Migration(migrations.Migration): ('completion_date', models.DateField(blank=True, null=True)), ('URL', models.URLField(blank=True, help_text='Link to external URL')), ('notes', models.TextField(blank=True, help_text='Extra build notes')), - ('completed_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='builds_completed', to=settings.AUTH_USER_MODEL)), - ('part', models.ForeignKey(help_text='Select part to build', limit_choices_to={'active': True, 'buildable': True}, on_delete=django.db.models.deletion.CASCADE, related_name='builds', to='part.Part')), - ('take_from', models.ForeignKey(blank=True, help_text='Select location to take stock from for this build (leave blank to take from any stock location)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='sourcing_builds', to='stock.StockLocation')), ], ), migrations.CreateModel( @@ -40,10 +33,6 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('quantity', models.PositiveIntegerField(default=1, help_text='Stock quantity to allocate to build', validators=[django.core.validators.MinValueValidator(1)])), ('build', models.ForeignKey(help_text='Build to allocate parts', on_delete=django.db.models.deletion.CASCADE, related_name='allocated_stock', to='build.Build')), - ('stock_item', models.ForeignKey(help_text='Stock Item to allocate to build', on_delete=django.db.models.deletion.CASCADE, related_name='allocations', to='stock.StockItem')), ], - options={ - 'unique_together': {('build', 'stock_item')}, - }, ), ] diff --git a/InvenTree/build/migrations/0002_auto_20190520_1922.py b/InvenTree/build/migrations/0002_auto_20190520_1922.py new file mode 100644 index 0000000000..1a20ff5bd4 --- /dev/null +++ b/InvenTree/build/migrations/0002_auto_20190520_1922.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2 on 2019-05-20 09:22 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('stock', '0001_initial'), + ('part', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('build', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='builditem', + name='stock_item', + field=models.ForeignKey(help_text='Stock Item to allocate to build', on_delete=django.db.models.deletion.CASCADE, related_name='allocations', to='stock.StockItem'), + ), + migrations.AddField( + model_name='build', + name='completed_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='builds_completed', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='build', + name='part', + field=models.ForeignKey(help_text='Select part to build', limit_choices_to={'active': True, 'buildable': True}, on_delete=django.db.models.deletion.CASCADE, related_name='builds', to='part.Part'), + ), + migrations.AddField( + model_name='build', + name='take_from', + field=models.ForeignKey(blank=True, help_text='Select location to take stock from for this build (leave blank to take from any stock location)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='sourcing_builds', to='stock.StockLocation'), + ), + migrations.AlterUniqueTogether( + name='builditem', + unique_together={('build', 'stock_item')}, + ), + ] diff --git a/InvenTree/company/migrations/0001_initial.py b/InvenTree/company/migrations/0001_initial.py index b4da1df89e..540b4bf47e 100644 --- a/InvenTree/company/migrations/0001_initial.py +++ b/InvenTree/company/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 import company.models import django.core.validators diff --git a/InvenTree/company/migrations/0002_auto_20190519_0004.py b/InvenTree/company/migrations/0002_auto_20190520_1922.py similarity index 96% rename from InvenTree/company/migrations/0002_auto_20190519_0004.py rename to InvenTree/company/migrations/0002_auto_20190520_1922.py index c34c315d48..df4c1c8910 100644 --- a/InvenTree/company/migrations/0002_auto_20190519_0004.py +++ b/InvenTree/company/migrations/0002_auto_20190520_1922.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 from django.db import migrations, models import django.db.models.deletion @@ -9,8 +9,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('company', '0001_initial'), ('part', '0001_initial'), + ('company', '0001_initial'), ] operations = [ diff --git a/InvenTree/part/migrations/0001_initial.py b/InvenTree/part/migrations/0001_initial.py index c0413a909a..cf7d7b8b98 100644 --- a/InvenTree/part/migrations/0001_initial.py +++ b/InvenTree/part/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 import InvenTree.validators from django.conf import settings diff --git a/InvenTree/part/migrations/0002_auto_20190519_0004.py b/InvenTree/part/migrations/0002_auto_20190520_1922.py similarity index 97% rename from InvenTree/part/migrations/0002_auto_20190519_0004.py rename to InvenTree/part/migrations/0002_auto_20190520_1922.py index 02df1a7fad..098176a44d 100644 --- a/InvenTree/part/migrations/0002_auto_20190519_0004.py +++ b/InvenTree/part/migrations/0002_auto_20190520_1922.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 from django.conf import settings from django.db import migrations, models @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('company', '0002_auto_20190519_0004'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('company', '0002_auto_20190520_1922'), ('stock', '0001_initial'), ('part', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/InvenTree/stock/migrations/0001_initial.py b/InvenTree/stock/migrations/0001_initial.py index 2fc4e96608..ea639842a9 100644 --- a/InvenTree/stock/migrations/0001_initial.py +++ b/InvenTree/stock/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 09:22 from django.conf import settings import django.core.validators @@ -11,9 +11,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('company', '0001_initial'), ('part', '0001_initial'), + ('company', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -70,7 +70,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='stockitem', name='part', - field=models.ForeignKey(help_text='Base part', on_delete=django.db.models.deletion.CASCADE, related_name='locations', to='part.Part'), + field=models.ForeignKey(help_text='Base part', on_delete=django.db.models.deletion.CASCADE, related_name='stock_items', to='part.Part'), ), migrations.AddField( model_name='stockitem', diff --git a/InvenTree/stock/models.py b/InvenTree/stock/models.py index fcb1622f27..3680079b30 100644 --- a/InvenTree/stock/models.py +++ b/InvenTree/stock/models.py @@ -186,7 +186,7 @@ class StockItem(models.Model): } ) - part = models.ForeignKey('part.Part', on_delete=models.CASCADE, related_name='locations', help_text='Base part') + part = models.ForeignKey('part.Part', on_delete=models.CASCADE, related_name='stock_items', help_text='Base part') supplier_part = models.ForeignKey('company.SupplierPart', blank=True, null=True, on_delete=models.SET_NULL, help_text='Select a matching supplier part for this stock item') From dd5de91c2306b4f73fd269f7b9dc7155e6cb68e2 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 21:46:50 +1000 Subject: [PATCH 08/20] Add middleware to report response time (and number of queries) --- InvenTree/InvenTree/middleware.py | 13 +++++++++++-- InvenTree/InvenTree/settings.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index c31a6e2b6d..1b8320c060 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -2,6 +2,7 @@ from django.shortcuts import HttpResponseRedirect from django.urls import reverse_lazy from django.db import connection import logging +import time logger = logging.getLogger(__name__) @@ -45,7 +46,9 @@ class QueryCountMiddleware(object): def __call__(self, request): + t_start = time.time() response = self.get_response(request) + t_stop = time.time() if response.status_code == 200: total_time = 0 @@ -54,14 +57,20 @@ class QueryCountMiddleware(object): for query in connection.queries: query_time = query.get('time') + if query_time is None: # django-debug-toolbar monkeypatches the connection # cursor wrapper and adds extra information in each # item in connection.queries. The query time is stored # under the key "duration" rather than "time" and is # in milliseconds, not seconds. - query_time = query.get('duration', 0) / 1000 + query_time = float(query.get('duration', 0)) + total_time += float(query_time) - logger.debug('%s queries run, total %s seconds' % (len(connection.queries), total_time)) + logger.debug('{n} queries run, {a:.3f}s / {b:.3f}s'.format( + n=len(connection.queries), + a=total_time, + b=(t_stop - t_start))) + return response diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index c6b8e88131..6e1acb4dd5 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -67,13 +67,38 @@ INSTALLED_APPS = [ 'django_filters', # Extended filter functionality 'dbbackup', # Database backup / restore 'rest_framework', # DRF (Django Rest Framework) - 'corsheaders', # Cross-origin Resource Sharing for DRF + 'corsheaders', # Cross-origin Resource Sharing for DRF 'crispy_forms', # Improved form rendering 'import_export', # Import / export tables to file 'django_cleanup', # Automatically delete orphaned MEDIA files 'qr_code', # Generate QR codes ] +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + }, + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': './debug.log', + }, + }, + + 'loggers': { + 'ddjango.db.backends': { + 'level': 'DEBUG', + 'handlers': ['file'], + 'propagate': True + }, + }, +} + + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -83,7 +108,8 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'InvenTree.middleware.AuthRequiredMiddleware' + 'InvenTree.middleware.AuthRequiredMiddleware', + 'InvenTree.middleware.QueryCountMiddleware', ] if DEBUG: From ee2ddbf512e95c6cac141dba0b2ae73a34e05580 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 21:47:30 +1000 Subject: [PATCH 09/20] Reduce some database queries by using aggregate Sum --- InvenTree/part/models.py | 18 +++++++++++++++--- InvenTree/part/serializers.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index d8b7529b23..569f373a3d 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -16,6 +16,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.db import models, transaction +from django.db.models import Sum from django.core.validators import MinValueValidator from django.contrib.staticfiles.templatetags.staticfiles import static @@ -481,7 +482,13 @@ class Part(models.Model): @property def stock_entries(self): - return [loc for loc in self.locations.all() if loc.in_stock] + """ Return all 'in stock' items. To be in stock: + + - customer is None + - belongs_to is None + """ + + return self.stock_items.filter(customer=None, belongs_to=None) @property def total_stock(self): @@ -489,7 +496,12 @@ class Part(models.Model): Part may be stored in multiple locations """ - return sum([loc.quantity for loc in self.stock_entries]) + total = self.stock_entries.aggregate(total=Sum('quantity'))['total'] + + if total: + return total + else: + return 0 @property def has_bom(self): @@ -518,7 +530,7 @@ class Part(models.Model): returns a string representation of a hash object which can be compared with a stored value """ - hash = hashlib.md5('bom seed'.encode()) + hash = hashlib.md5(str(item.part.id).encode()) for item in self.bom_items.all(): hash.update(str(item.sub_part.id).encode()) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index d4eb82d0a1..c847785476 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -61,7 +61,7 @@ class PartSerializer(serializers.ModelSerializer): @staticmethod def setup_eager_loading(queryset): queryset = queryset.prefetch_related('category') - queryset = queryset.prefetch_related('locations') + queryset = queryset.prefetch_related('stock_items') return queryset class Meta: From 4bde00d9254f53704aacb52af3d719850f87145b Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 22:24:18 +1000 Subject: [PATCH 10/20] AND AGAIN Use --fake to force the migration --- InvenTree/build/migrations/0001_initial.py | 2 +- ...002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} | 6 +++--- InvenTree/company/migrations/0001_initial.py | 2 +- ...002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} | 2 +- InvenTree/part/migrations/0001_initial.py | 2 +- ...002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} | 6 +++--- InvenTree/stock/migrations/0001_initial.py | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) rename InvenTree/build/migrations/{0002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} (97%) rename InvenTree/company/migrations/{0002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} (96%) rename InvenTree/part/migrations/{0002_auto_20190520_1922.py => 0002_auto_20190520_2204.py} (97%) diff --git a/InvenTree/build/migrations/0001_initial.py b/InvenTree/build/migrations/0001_initial.py index 85fcb1b827..6e55be5678 100644 --- a/InvenTree/build/migrations/0001_initial.py +++ b/InvenTree/build/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 import django.core.validators from django.db import migrations, models diff --git a/InvenTree/build/migrations/0002_auto_20190520_1922.py b/InvenTree/build/migrations/0002_auto_20190520_2204.py similarity index 97% rename from InvenTree/build/migrations/0002_auto_20190520_1922.py rename to InvenTree/build/migrations/0002_auto_20190520_2204.py index 1a20ff5bd4..974d4bfc5a 100644 --- a/InvenTree/build/migrations/0002_auto_20190520_1922.py +++ b/InvenTree/build/migrations/0002_auto_20190520_2204.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.conf import settings from django.db import migrations, models @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('stock', '0001_initial'), ('part', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('build', '0001_initial'), + ('stock', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/InvenTree/company/migrations/0001_initial.py b/InvenTree/company/migrations/0001_initial.py index 540b4bf47e..c2de9ed453 100644 --- a/InvenTree/company/migrations/0001_initial.py +++ b/InvenTree/company/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 import company.models import django.core.validators diff --git a/InvenTree/company/migrations/0002_auto_20190520_1922.py b/InvenTree/company/migrations/0002_auto_20190520_2204.py similarity index 96% rename from InvenTree/company/migrations/0002_auto_20190520_1922.py rename to InvenTree/company/migrations/0002_auto_20190520_2204.py index df4c1c8910..4c58d24989 100644 --- a/InvenTree/company/migrations/0002_auto_20190520_1922.py +++ b/InvenTree/company/migrations/0002_auto_20190520_2204.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.db import migrations, models import django.db.models.deletion diff --git a/InvenTree/part/migrations/0001_initial.py b/InvenTree/part/migrations/0001_initial.py index cf7d7b8b98..e5183e2b5c 100644 --- a/InvenTree/part/migrations/0001_initial.py +++ b/InvenTree/part/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 import InvenTree.validators from django.conf import settings diff --git a/InvenTree/part/migrations/0002_auto_20190520_1922.py b/InvenTree/part/migrations/0002_auto_20190520_2204.py similarity index 97% rename from InvenTree/part/migrations/0002_auto_20190520_1922.py rename to InvenTree/part/migrations/0002_auto_20190520_2204.py index 098176a44d..cf968b43cd 100644 --- a/InvenTree/part/migrations/0002_auto_20190520_1922.py +++ b/InvenTree/part/migrations/0002_auto_20190520_2204.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.conf import settings from django.db import migrations, models @@ -10,9 +10,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('company', '0002_auto_20190520_1922'), - ('stock', '0001_initial'), ('part', '0001_initial'), + ('company', '0002_auto_20190520_2204'), + ('stock', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] diff --git a/InvenTree/stock/migrations/0001_initial.py b/InvenTree/stock/migrations/0001_initial.py index ea639842a9..09c033c06b 100644 --- a/InvenTree/stock/migrations/0001_initial.py +++ b/InvenTree/stock/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-20 09:22 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.conf import settings import django.core.validators From 18ff80312b6cd9c09974542da00e124d1f870b1e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 22:53:01 +1000 Subject: [PATCH 11/20] Fixes --- InvenTree/part/models.py | 4 ++-- InvenTree/part/serializers.py | 4 +++- InvenTree/stock/serializers.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 569f373a3d..6d6f02f6a3 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -463,7 +463,7 @@ class Part(models.Model): @property def allocated_build_count(self): - """ Return the total number of this that are allocated for builds + """ Return the total number of this part that are allocated for builds """ return sum([a['quantity'] for a in self.build_allocation]) @@ -530,7 +530,7 @@ class Part(models.Model): returns a string representation of a hash object which can be compared with a stored value """ - hash = hashlib.md5(str(item.part.id).encode()) + hash = hashlib.md5(str(self.id).encode()) for item in self.bom_items.all(): hash.update(str(item.sub_part.id).encode()) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index c847785476..e91a3c6589 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -112,6 +112,7 @@ class PartStarSerializer(InvenTreeModelSerializer): class BomItemSerializer(InvenTreeModelSerializer): """ Serializer for BomItem object """ + part_detail = PartBriefSerializer(source='part', many=False, read_only=True) sub_part_detail = PartBriefSerializer(source='sub_part', many=False, read_only=True) price_info = serializers.CharField(read_only=True) @@ -119,7 +120,7 @@ class BomItemSerializer(InvenTreeModelSerializer): def setup_eager_loading(queryset): queryset = queryset.prefetch_related('sub_part') queryset = queryset.prefetch_related('sub_part__category') - queryset = queryset.prefetch_related('sub_part__locations') + queryset = queryset.prefetch_related('sub_part__stock_items') return queryset class Meta: @@ -127,6 +128,7 @@ class BomItemSerializer(InvenTreeModelSerializer): fields = [ 'pk', 'part', + 'part_detail', 'sub_part', 'sub_part_detail', 'quantity', diff --git a/InvenTree/stock/serializers.py b/InvenTree/stock/serializers.py index 46a1c44845..40697a0aee 100644 --- a/InvenTree/stock/serializers.py +++ b/InvenTree/stock/serializers.py @@ -63,7 +63,7 @@ class StockItemSerializer(serializers.ModelSerializer): @staticmethod def setup_eager_loading(queryset): queryset = queryset.prefetch_related('part') - queryset = queryset.prefetch_related('part__locations') + queryset = queryset.prefetch_related('part__stock_items') queryset = queryset.prefetch_related('part__category') queryset = queryset.prefetch_related('location') From d15b09a5f4eddccc1f69146adcea3a39bb813fa1 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 23:03:19 +1000 Subject: [PATCH 12/20] Don't serialize pricing - Saves MANY queries and provides drastic speedup --- InvenTree/part/models.py | 5 ----- InvenTree/part/serializers.py | 4 ---- InvenTree/static/script/inventree/bom.js | 13 ------------- 3 files changed, 22 deletions(-) diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 6d6f02f6a3..19b2543da2 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -1007,8 +1007,3 @@ class BomItem(models.Model): base_quantity = self.quantity * build_quantity return base_quantity + self.get_overage_quantity(base_quantity) - - @property - def price_info(self): - """ Return the price for this item in the BOM """ - return self.sub_part.get_price_info(self.quantity) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index e91a3c6589..d8da916cb2 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -34,7 +34,6 @@ class PartBriefSerializer(serializers.ModelSerializer): url = serializers.CharField(source='get_absolute_url', read_only=True) image_url = serializers.CharField(source='get_image_url', read_only=True) - single_price_info = serializers.CharField(read_only=True) class Meta: model = Part @@ -44,7 +43,6 @@ class PartBriefSerializer(serializers.ModelSerializer): 'full_name', 'description', 'available_stock', - 'single_price_info', 'image_url', ] @@ -114,7 +112,6 @@ class BomItemSerializer(InvenTreeModelSerializer): part_detail = PartBriefSerializer(source='part', many=False, read_only=True) sub_part_detail = PartBriefSerializer(source='sub_part', many=False, read_only=True) - price_info = serializers.CharField(read_only=True) @staticmethod def setup_eager_loading(queryset): @@ -132,7 +129,6 @@ class BomItemSerializer(InvenTreeModelSerializer): 'sub_part', 'sub_part_detail', 'quantity', - 'price_info', 'overage', 'note', ] diff --git a/InvenTree/static/script/inventree/bom.js b/InvenTree/static/script/inventree/bom.js index 6e00eabe67..6ff81de4fc 100644 --- a/InvenTree/static/script/inventree/bom.js +++ b/InvenTree/static/script/inventree/bom.js @@ -151,19 +151,6 @@ function loadBomTable(table, options) { } } ); - - cols.push({ - field: 'price_info', - title: 'Price', - sortable: true, - formatter: function(value, row, index, field) { - if (value) { - return value; - } else { - return "No pricing information"; - } - }, - }); } // Part notes From 8adb4f6c204795b116f0c9c18e4ec8b2cf127626 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 23:26:27 +1000 Subject: [PATCH 13/20] Further query reduction - Also improved query reporting middleware --- InvenTree/InvenTree/middleware.py | 13 +++++++++++++ InvenTree/part/serializers.py | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index 1b8320c060..bf4e5e990b 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -3,6 +3,7 @@ from django.urls import reverse_lazy from django.db import connection import logging import time +import operator logger = logging.getLogger(__name__) @@ -55,9 +56,18 @@ class QueryCountMiddleware(object): if len(connection.queries) > 0: + queries = {} + for query in connection.queries: query_time = query.get('time') + sql = query.get('sql').split('.')[0] + + if sql in queries: + queries[sql] += 1 + else: + queries[sql] = 1 + if query_time is None: # django-debug-toolbar monkeypatches the connection # cursor wrapper and adds extra information in each @@ -73,4 +83,7 @@ class QueryCountMiddleware(object): a=total_time, b=(t_stop - t_start))) + for x in sorted(queries.items(), key=operator.itemgetter(1), reverse=True): + print(x[0], ':', x[1]) + return response diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index d8da916cb2..6566ba52ad 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -34,6 +34,14 @@ class PartBriefSerializer(serializers.ModelSerializer): url = serializers.CharField(source='get_absolute_url', read_only=True) image_url = serializers.CharField(source='get_image_url', read_only=True) + + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('category') + queryset = queryset.prefetch_related('stock_items') + queryset = queryset.prefetch_related('bom_items') + queryset = queryset.prefetch_related('builds') + return queryset class Meta: model = Part @@ -60,6 +68,8 @@ class PartSerializer(serializers.ModelSerializer): def setup_eager_loading(queryset): queryset = queryset.prefetch_related('category') queryset = queryset.prefetch_related('stock_items') + queryset = queryset.prefetch_related('bom_items') + queryset = queryset.prefetch_related('builds') return queryset class Meta: From 6ae48d07c479c2fae2bd3bfb65a5a53a25e05fad Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 20 May 2019 23:53:39 +1000 Subject: [PATCH 14/20] Cleanup pricing algorithms --- InvenTree/company/models.py | 2 +- InvenTree/part/models.py | 170 +++++++++++------------------------- InvenTree/part/views.py | 38 ++++---- 3 files changed, 71 insertions(+), 139 deletions(-) diff --git a/InvenTree/company/models.py b/InvenTree/company/models.py index 90a0df23a2..be6cf10f7c 100644 --- a/InvenTree/company/models.py +++ b/InvenTree/company/models.py @@ -250,7 +250,7 @@ class SupplierPart(models.Model): - If order multiples are to be observed, then we need to calculate based on that, too """ - price_breaks = self.price_breaks.all() + price_breaks = self.price_breaks.filter(quantity__lte=quantity) # No price break information available? if len(price_breaks) == 0: diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 19b2543da2..1afb629437 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -573,32 +573,16 @@ class Part(models.Model): """ Return the number of supplier parts available for this part """ return self.supplier_parts.count() - @property - def min_single_price(self): - return self.get_min_supplier_price(1) - - @property - def max_single_price(self): - return self.get_max_supplier_price(1) - - @property - def min_bom_price(self): - return self.get_min_bom_price(1) - - @property - def max_bom_price(self): - return self.get_max_bom_price(1) - @property def has_pricing_info(self): """ Return true if there is pricing information for this part """ - return self.get_min_price() is not None + return self.get_price_range() is not None @property def has_complete_bom_pricing(self): """ Return true if there is pricing information for each item in the BOM. """ - for item in self.bom_items.all(): + for item in self.bom_items.all().prefetch_related('sub_part'): if not item.sub_part.has_pricing_info: return False @@ -618,72 +602,45 @@ class Part(models.Model): buy: Include supplier pricing (default = True) bom: Include BOM pricing (default = True) """ - - min_price = self.get_min_price(quantity, buy, bom) - max_price = self.get_max_price(quantity, buy, bom) - if min_price is None: + price_range = self.get_price_range(quantity, buy, bom) + + if price_range is None: return None + min_price, max_price = price_range + if min_price == max_price: return min_price return "{a} to {b}".format(a=min_price, b=max_price) - def get_min_supplier_price(self, quantity=1): - """ Return the minimum price of this part from all available suppliers. + def get_supplier_price_range(self, quantity=1): - Args: - quantity: Number of units we wish to purchase (default = 1) - - Returns: - Numerical price if pricing is available, else None - """ - min_price = None - - for supplier_part in self.supplier_parts.all(): - supplier_price = supplier_part.get_price(quantity) - - if supplier_price is None: - continue - - if min_price is None or supplier_price < min_price: - min_price = supplier_price - - if min_price is None: - return None - else: - return min_price - - def get_max_supplier_price(self, quantity=1): - """ Return the maximum price of this part from all available suppliers. - - Args: - quantity: Number of units we wish to purchase (default = 1) - - Returns: - Numerical price if pricing is available, else None - """ - max_price = None - for supplier_part in self.supplier_parts.all(): - supplier_price = supplier_part.get_price(quantity) + for supplier in self.supplier_parts.all(): - if supplier_price is None: + price = supplier.get_price(quantity) + + if price is None: continue - if max_price is None or supplier_price > max_price: - max_price = supplier_price + if min_price is None or price < min_price: + min_price = price - if max_price is None: + if max_price is None or price > max_price: + max_price = price + + if min_price is None or max_price is None: return None - else: - return max_price - def get_min_bom_price(self, quantity=1): - """ Return the minimum price of the BOM for this part. + return (min_price, max_price) + + + def get_bom_price_range(self, quantity=1): + """ Return the price range of the BOM for this part. Adds the minimum price for all components in the BOM. Note: If the BOM contains items without pricing information, @@ -691,45 +648,33 @@ class Part(models.Model): """ min_price = None + max_price = None - for item in self.bom_items.all(): - price = item.sub_part.get_min_price(quantity * item.quantity) + for item in self.bom_items.all().prefetch_related('sub_part'): + prices = item.sub_part.get_price_range(quantity * item.quantity) - if price is None: + if prices is None: continue + low, high = prices + if min_price is None: min_price = 0 - min_price += price - - return min_price - - def get_max_bom_price(self, quantity=1): - """ Return the maximum price of the BOM for this part. - Adds the maximum price for all components in the BOM. - - Note: If the BOM contains items without pricing information, - these items cannot be included in the BOM! - """ - - max_price = None - - for item in self.bom_items.all(): - price = item.sub_part.get_max_price(quantity * item.quantity) - - if price is None: - continue - if max_price is None: max_price = 0 - max_price += price + min_price += low + max_price += high - return max_price + if min_price is None or max_price is None: + return None - def get_min_price(self, quantity=1, buy=True, bom=True): - """ Return the minimum price for this part. This price can be either: + return (min_price, max_price) + + def get_price_range(self, quantity=1, buy=True, bom=True): + + """ Return the price range for this part. This price can be either: - Supplier price (if purchased from suppliers) - BOM price (if built from other parts) @@ -738,37 +683,20 @@ class Part(models.Model): Minimum of the supplier price or BOM price. If no pricing available, returns None """ - buy_price = self.get_min_supplier_price(quantity) if buy else None - bom_price = self.get_min_bom_price(quantity) if bom else None + buy_price_range = self.get_supplier_price_range(quantity) if buy else None + bom_price_range = self.get_bom_price_range(quantity) if bom else None - if buy_price is None: - return bom_price + if buy_price_range is None: + return bom_price_range - if bom_price is None: - return buy_price - - return min(buy_price, bom_price) + elif bom_price_range is None: + return buy_price_range - def get_max_price(self, quantity=1, buy=True, bom=True): - """ Return the maximum price for this part. This price can be either: - - - Supplier price (if purchsed from suppliers) - - BOM price (if built from other parts) - - Returns: - Maximum of the supplier price or BOM price. If no pricing available, returns None - """ - - buy_price = self.get_max_supplier_price(quantity) if buy else None - bom_price = self.get_max_bom_price(quantity) if bom else None - - if buy_price is None: - return bom_price - - if bom_price is None: - return buy_price - - return max(buy_price, bom_price) + else: + return ( + min(buy_price_range[0], bom_price_range[0]), + max(buy_price_range[1], bom_price_range[1]) + ) def deepCopy(self, other, **kwargs): """ Duplicates non-field data from another part. diff --git a/InvenTree/part/views.py b/InvenTree/part/views.py index 98a209b714..89065a2afd 100644 --- a/InvenTree/part/views.py +++ b/InvenTree/part/views.py @@ -587,30 +587,34 @@ class PartPricing(AjaxView): # Supplier pricing information if part.supplier_count > 0: - min_buy_price = part.get_min_supplier_price(quantity) - max_buy_price = part.get_max_supplier_price(quantity) + buy_price = part.get_supplier_price_range(quantity) - if min_buy_price: - ctx['min_total_buy_price'] = min_buy_price - ctx['min_unit_buy_price'] = min_buy_price / quantity + if buy_price is not None: + min_buy_price, max_buy_price = buy_price - if max_buy_price: - ctx['max_total_buy_price'] = max_buy_price - ctx['max_unit_buy_price'] = max_buy_price / quantity + if min_buy_price: + ctx['min_total_buy_price'] = min_buy_price + ctx['min_unit_buy_price'] = min_buy_price / quantity + + if max_buy_price: + ctx['max_total_buy_price'] = max_buy_price + ctx['max_unit_buy_price'] = max_buy_price / quantity # BOM pricing information if part.bom_count > 0: - min_bom_price = part.get_min_bom_price(quantity) - max_bom_price = part.get_max_bom_price(quantity) + bom_price = part.get_bom_price_range(quantity) - if min_bom_price: - ctx['min_total_bom_price'] = min_bom_price - ctx['min_unit_bom_price'] = min_bom_price / quantity - - if max_bom_price: - ctx['max_total_bom_price'] = max_bom_price - ctx['max_unit_bom_price'] = max_bom_price / quantity + if bom_price is not None: + min_bom_price, max_bom_price = bom_price + + if min_bom_price: + ctx['min_total_bom_price'] = min_bom_price + ctx['min_unit_bom_price'] = min_bom_price / quantity + + if max_bom_price: + ctx['max_total_bom_price'] = max_bom_price + ctx['max_unit_bom_price'] = max_bom_price / quantity return ctx From 157919f47af66f2860d1fd82bd76d636e2b88f89 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:06:57 +1000 Subject: [PATCH 15/20] More prefetching - --- InvenTree/InvenTree/middleware.py | 4 +++- InvenTree/part/models.py | 20 +++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index bf4e5e990b..9f1278d391 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -82,8 +82,10 @@ class QueryCountMiddleware(object): n=len(connection.queries), a=total_time, b=(t_stop - t_start))) - + + """ for x in sorted(queries.items(), key=operator.itemgetter(1), reverse=True): print(x[0], ':', x[1]) + """ return response diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 1afb629437..6f1563ef8b 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -411,7 +411,7 @@ class Part(models.Model): total = None # Calculate the minimum number of parts that can be built using each sub-part - for item in self.bom_items.all(): + for item in self.bom_items.all().select_related('sub_part'): stock = item.sub_part.available_stock n = int(1.0 * stock / item.quantity) @@ -449,7 +449,7 @@ class Part(models.Model): builds = [] - for item in self.used_in.all(): + for item in self.used_in.all().prefetch_related('part'): for build in item.part.active_builds: b = {} @@ -532,7 +532,7 @@ class Part(models.Model): hash = hashlib.md5(str(self.id).encode()) - for item in self.bom_items.all(): + for item in self.bom_items.all().prefetch('sub_part'): hash.update(str(item.sub_part.id).encode()) hash.update(str(item.sub_part.full_name).encode()) hash.update(str(item.quantity).encode()) @@ -564,7 +564,7 @@ class Part(models.Model): def required_parts(self): """ Return a list of parts required to make this part (list of BOM items) """ parts = [] - for bom in self.bom_items.all(): + for bom in self.bom_items.all().select_related('sub_part'): parts.append(bom.sub_part) return parts @@ -582,18 +582,12 @@ class Part(models.Model): def has_complete_bom_pricing(self): """ Return true if there is pricing information for each item in the BOM. """ - for item in self.bom_items.all().prefetch_related('sub_part'): + for item in self.bom_items.all().select_related('sub_part'): if not item.sub_part.has_pricing_info: return False return True - @property - def single_price_info(self): - """ Return a simplified pricing string for this part at single quantity """ - - return self.get_price_info() - def get_price_info(self, quantity=1, buy=True, bom=True): """ Return a simplified pricing string for this part @@ -613,7 +607,7 @@ class Part(models.Model): if min_price == max_price: return min_price - return "{a} to {b}".format(a=min_price, b=max_price) + return "{a} - {b}".format(a=min_price, b=max_price) def get_supplier_price_range(self, quantity=1): @@ -650,7 +644,7 @@ class Part(models.Model): min_price = None max_price = None - for item in self.bom_items.all().prefetch_related('sub_part'): + for item in self.bom_items.all().select_related('sub_part'): prices = item.sub_part.get_price_range(quantity * item.quantity) if prices is None: From 72aba30e8190a43a81f28354e630ddf573a313e3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:16:00 +1000 Subject: [PATCH 16/20] More hungry fetching --- InvenTree/build/models.py | 6 +++--- InvenTree/part/models.py | 2 +- InvenTree/part/serializers.py | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/InvenTree/build/models.py b/InvenTree/build/models.py index c69857fda3..92f04bdcc6 100644 --- a/InvenTree/build/models.py +++ b/InvenTree/build/models.py @@ -139,7 +139,7 @@ class Build(models.Model): allocations = [] - for item in self.part.bom_items.all(): + for item in self.part.bom_items.all().prefetch_related('sub_part'): # How many parts required for this build? q_required = item.quantity * self.quantity @@ -216,7 +216,7 @@ class Build(models.Model): - Delete pending BuildItem objects """ - for item in self.allocated_stock.all(): + for item in self.allocated_stock.all().prefetch_related('stock_item'): # Subtract stock from the item item.stock_item.take_stock( @@ -295,7 +295,7 @@ class Build(models.Model): """ Returns a dict of parts required to build this part (BOM) """ parts = [] - for item in self.part.bom_items.all(): + for item in self.part.bom_items.all().prefetch_related('sub_part'): part = {'part': item.sub_part, 'per_build': item.quantity, 'quantity': item.quantity * self.quantity, diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 6f1563ef8b..46afc5d1f1 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -532,7 +532,7 @@ class Part(models.Model): hash = hashlib.md5(str(self.id).encode()) - for item in self.bom_items.all().prefetch('sub_part'): + for item in self.bom_items.all().prefetch_related('sub_part'): hash.update(str(item.sub_part.id).encode()) hash.update(str(item.sub_part.full_name).encode()) hash.update(str(item.quantity).encode()) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 6566ba52ad..ddbbab9e57 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -125,6 +125,9 @@ class BomItemSerializer(InvenTreeModelSerializer): @staticmethod def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('part') + queryset = queryset.prefetch_related('part__category') + queryset = queryset.prefetch_related('part__stock_items') queryset = queryset.prefetch_related('sub_part') queryset = queryset.prefetch_related('sub_part__category') queryset = queryset.prefetch_related('sub_part__stock_items') From 4b41766312082d338bf6cb458ddfddf166acd2c3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:31:34 +1000 Subject: [PATCH 17/20] And mode --- InvenTree/part/templates/part/bom.html | 3 --- InvenTree/part/views.py | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/InvenTree/part/templates/part/bom.html b/InvenTree/part/templates/part/bom.html index bff082b6ae..19b876d905 100644 --- a/InvenTree/part/templates/part/bom.html +++ b/InvenTree/part/templates/part/bom.html @@ -16,9 +16,6 @@ The BOM for {{ part.full_name }} does not have complete pricing information {% endif %} -
- Single BOM Price: {{ part.min_bom_price }} to {{ part.max_bom_price }} -
{% if part.bom_checked_date %} {% if part.is_bom_valid %}
diff --git a/InvenTree/part/views.py b/InvenTree/part/views.py index 89065a2afd..02b5b19909 100644 --- a/InvenTree/part/views.py +++ b/InvenTree/part/views.py @@ -34,7 +34,7 @@ class PartIndex(ListView): context_object_name = 'parts' def get_queryset(self): - return Part.objects.all() # filter(category=None) + return Part.objects.all().select_related('category') def get_context_data(self, **kwargs): @@ -355,7 +355,7 @@ class PartDetail(DetailView): """ context_object_name = 'part' - queryset = Part.objects.all() + queryset = Part.objects.all().select_related('category') template_name = 'part/detail.html' # Add in some extra context information based on query params @@ -641,7 +641,7 @@ class CategoryDetail(DetailView): """ Detail view for PartCategory """ model = PartCategory context_object_name = 'category' - queryset = PartCategory.objects.all() + queryset = PartCategory.objects.all().prefetch_related('children') template_name = 'part/category.html' From ae865d1de153404f8491df6532097581341eddd0 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:40:42 +1000 Subject: [PATCH 18/20] Commit --- InvenTree/InvenTree/middleware.py | 2 -- InvenTree/InvenTree/settings.py | 8 +++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index 9f1278d391..9abf3643e3 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -83,9 +83,7 @@ class QueryCountMiddleware(object): a=total_time, b=(t_stop - t_start))) - """ for x in sorted(queries.items(), key=operator.itemgetter(1), reverse=True): print(x[0], ':', x[1]) - """ return response diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 6e1acb4dd5..3a42b7d163 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -82,17 +82,19 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, - 'file': { + 'debug_file': { 'level': 'DEBUG', 'class': 'logging.FileHandler', - 'filename': './debug.log', + 'filename': os.path.join(BASE_DIR, 'debug.log'), + #'maxBytes': 1024*1024*15, # 15MB + #'backupCount': 10, }, }, 'loggers': { 'ddjango.db.backends': { 'level': 'DEBUG', - 'handlers': ['file'], + 'handlers': ['debug_file', 'console',], 'propagate': True }, }, From 49ed17db56df9f26de47cdf00e2960995700eed3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:52:17 +1000 Subject: [PATCH 19/20] Add a line --- InvenTree/part/templates/part/part_app_base.html | 1 + 1 file changed, 1 insertion(+) diff --git a/InvenTree/part/templates/part/part_app_base.html b/InvenTree/part/templates/part/part_app_base.html index adf9d67617..da50dc5b55 100644 --- a/InvenTree/part/templates/part/part_app_base.html +++ b/InvenTree/part/templates/part/part_app_base.html @@ -34,6 +34,7 @@ InvenTree | Part List {% block js_ready %} {{ block.super }} + loadTree("{% url 'api-part-tree' %}", "#part-tree", { From ca9d6e6e25ee093585129ab4edaf3f451bbf1571 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 21 May 2019 00:54:48 +1000 Subject: [PATCH 20/20] PEP fixes --- InvenTree/InvenTree/settings.py | 21 +++++++-------------- InvenTree/part/models.py | 3 +-- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 3a42b7d163..058920c134 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -82,22 +82,15 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, - 'debug_file': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': os.path.join(BASE_DIR, 'debug.log'), - #'maxBytes': 1024*1024*15, # 15MB - #'backupCount': 10, - }, }, - 'loggers': { - 'ddjango.db.backends': { - 'level': 'DEBUG', - 'handlers': ['debug_file', 'console',], - 'propagate': True - }, - }, + # 'loggers': { + # 'ddjango.db.backends': { + # 'level': 'DEBUG', + # 'handlers': ['console',], + # 'propagate': True + # }, + # }, } diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 46afc5d1f1..04277508c7 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -632,7 +632,6 @@ class Part(models.Model): return (min_price, max_price) - def get_bom_price_range(self, quantity=1): """ Return the price range of the BOM for this part. Adds the minimum price for all components in the BOM. @@ -690,7 +689,7 @@ class Part(models.Model): return ( min(buy_price_range[0], bom_price_range[0]), max(buy_price_range[1], bom_price_range[1]) - ) + ) def deepCopy(self, other, **kwargs): """ Duplicates non-field data from another part.