diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index 3a5c4f059f..9abf3643e3 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -1,5 +1,11 @@ from django.shortcuts import HttpResponseRedirect from django.urls import reverse_lazy +from django.db import connection +import logging +import time +import operator + +logger = logging.getLogger(__name__) class AuthRequiredMiddleware(object): @@ -24,3 +30,60 @@ class AuthRequiredMiddleware(object): # the view is called. return response + + +class QueryCountMiddleware(object): + """ + This middleware will log the number of queries run + and the total time taken for each request (with a + status code of 200). It does not currently support + multi-db setups. + + Reference: https://www.dabapps.com/blog/logging-sql-queries-django-13/ + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + + t_start = time.time() + response = self.get_response(request) + t_stop = time.time() + + if response.status_code == 200: + total_time = 0 + + if len(connection.queries) > 0: + + queries = {} + + for query in connection.queries: + query_time = query.get('time') + + sql = query.get('sql').split('.')[0] + + if sql in queries: + queries[sql] += 1 + else: + queries[sql] = 1 + + if query_time is None: + # django-debug-toolbar monkeypatches the connection + # cursor wrapper and adds extra information in each + # item in connection.queries. The query time is stored + # under the key "duration" rather than "time" and is + # in milliseconds, not seconds. + query_time = float(query.get('duration', 0)) + + total_time += float(query_time) + + logger.debug('{n} queries run, {a:.3f}s / {b:.3f}s'.format( + n=len(connection.queries), + a=total_time, + b=(t_stop - t_start))) + + for x in sorted(queries.items(), key=operator.itemgetter(1), reverse=True): + print(x[0], ':', x[1]) + + return response diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 164050d5ca..058920c134 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -67,13 +67,33 @@ INSTALLED_APPS = [ 'django_filters', # Extended filter functionality 'dbbackup', # Database backup / restore 'rest_framework', # DRF (Django Rest Framework) - 'corsheaders', # Cross-origin Resource Sharing for DRF + 'corsheaders', # Cross-origin Resource Sharing for DRF 'crispy_forms', # Improved form rendering 'import_export', # Import / export tables to file 'django_cleanup', # Automatically delete orphaned MEDIA files 'qr_code', # Generate QR codes ] +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + }, + }, + + # 'loggers': { + # 'ddjango.db.backends': { + # 'level': 'DEBUG', + # 'handlers': ['console',], + # 'propagate': True + # }, + # }, +} + + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -83,9 +103,13 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'InvenTree.middleware.AuthRequiredMiddleware' + 'InvenTree.middleware.AuthRequiredMiddleware', + 'InvenTree.middleware.QueryCountMiddleware', ] +if DEBUG: + MIDDLEWARE.append('InvenTree.middleware.QueryCountMiddleware') + ROOT_URLCONF = 'InvenTree.urls' TEMPLATES = [ diff --git a/InvenTree/build/migrations/0001_initial.py b/InvenTree/build/migrations/0001_initial.py index aa7356389a..6e55be5678 100644 --- a/InvenTree/build/migrations/0001_initial.py +++ b/InvenTree/build/migrations/0001_initial.py @@ -1,6 +1,5 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 12:04 -from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion @@ -11,9 +10,6 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('part', '0002_auto_20190519_0004'), - ('stock', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -29,9 +25,6 @@ class Migration(migrations.Migration): ('completion_date', models.DateField(blank=True, null=True)), ('URL', models.URLField(blank=True, help_text='Link to external URL')), ('notes', models.TextField(blank=True, help_text='Extra build notes')), - ('completed_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='builds_completed', to=settings.AUTH_USER_MODEL)), - ('part', models.ForeignKey(help_text='Select part to build', limit_choices_to={'active': True, 'buildable': True}, on_delete=django.db.models.deletion.CASCADE, related_name='builds', to='part.Part')), - ('take_from', models.ForeignKey(blank=True, help_text='Select location to take stock from for this build (leave blank to take from any stock location)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='sourcing_builds', to='stock.StockLocation')), ], ), migrations.CreateModel( @@ -40,10 +33,6 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('quantity', models.PositiveIntegerField(default=1, help_text='Stock quantity to allocate to build', validators=[django.core.validators.MinValueValidator(1)])), ('build', models.ForeignKey(help_text='Build to allocate parts', on_delete=django.db.models.deletion.CASCADE, related_name='allocated_stock', to='build.Build')), - ('stock_item', models.ForeignKey(help_text='Stock Item to allocate to build', on_delete=django.db.models.deletion.CASCADE, related_name='allocations', to='stock.StockItem')), ], - options={ - 'unique_together': {('build', 'stock_item')}, - }, ), ] diff --git a/InvenTree/build/migrations/0002_auto_20190520_2204.py b/InvenTree/build/migrations/0002_auto_20190520_2204.py new file mode 100644 index 0000000000..974d4bfc5a --- /dev/null +++ b/InvenTree/build/migrations/0002_auto_20190520_2204.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2 on 2019-05-20 12:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('part', '0001_initial'), + ('build', '0001_initial'), + ('stock', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='builditem', + name='stock_item', + field=models.ForeignKey(help_text='Stock Item to allocate to build', on_delete=django.db.models.deletion.CASCADE, related_name='allocations', to='stock.StockItem'), + ), + migrations.AddField( + model_name='build', + name='completed_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='builds_completed', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='build', + name='part', + field=models.ForeignKey(help_text='Select part to build', limit_choices_to={'active': True, 'buildable': True}, on_delete=django.db.models.deletion.CASCADE, related_name='builds', to='part.Part'), + ), + migrations.AddField( + model_name='build', + name='take_from', + field=models.ForeignKey(blank=True, help_text='Select location to take stock from for this build (leave blank to take from any stock location)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='sourcing_builds', to='stock.StockLocation'), + ), + migrations.AlterUniqueTogether( + name='builditem', + unique_together={('build', 'stock_item')}, + ), + ] diff --git a/InvenTree/build/models.py b/InvenTree/build/models.py index c69857fda3..92f04bdcc6 100644 --- a/InvenTree/build/models.py +++ b/InvenTree/build/models.py @@ -139,7 +139,7 @@ class Build(models.Model): allocations = [] - for item in self.part.bom_items.all(): + for item in self.part.bom_items.all().prefetch_related('sub_part'): # How many parts required for this build? q_required = item.quantity * self.quantity @@ -216,7 +216,7 @@ class Build(models.Model): - Delete pending BuildItem objects """ - for item in self.allocated_stock.all(): + for item in self.allocated_stock.all().prefetch_related('stock_item'): # Subtract stock from the item item.stock_item.take_stock( @@ -295,7 +295,7 @@ class Build(models.Model): """ Returns a dict of parts required to build this part (BOM) """ parts = [] - for item in self.part.bom_items.all(): + for item in self.part.bom_items.all().prefetch_related('sub_part'): part = {'part': item.sub_part, 'per_build': item.quantity, 'quantity': item.quantity * self.quantity, diff --git a/InvenTree/company/migrations/0001_initial.py b/InvenTree/company/migrations/0001_initial.py index b4da1df89e..c2de9ed453 100644 --- a/InvenTree/company/migrations/0001_initial.py +++ b/InvenTree/company/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 12:04 import company.models import django.core.validators diff --git a/InvenTree/company/migrations/0002_auto_20190519_0004.py b/InvenTree/company/migrations/0002_auto_20190520_2204.py similarity index 96% rename from InvenTree/company/migrations/0002_auto_20190519_0004.py rename to InvenTree/company/migrations/0002_auto_20190520_2204.py index c34c315d48..4c58d24989 100644 --- a/InvenTree/company/migrations/0002_auto_20190519_0004.py +++ b/InvenTree/company/migrations/0002_auto_20190520_2204.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.db import migrations, models import django.db.models.deletion @@ -9,8 +9,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('company', '0001_initial'), ('part', '0001_initial'), + ('company', '0001_initial'), ] operations = [ diff --git a/InvenTree/company/models.py b/InvenTree/company/models.py index 90a0df23a2..be6cf10f7c 100644 --- a/InvenTree/company/models.py +++ b/InvenTree/company/models.py @@ -250,7 +250,7 @@ class SupplierPart(models.Model): - If order multiples are to be observed, then we need to calculate based on that, too """ - price_breaks = self.price_breaks.all() + price_breaks = self.price_breaks.filter(quantity__lte=quantity) # No price break information available? if len(price_breaks) == 0: diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index aa065e9720..f3da5ccf40 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -110,6 +110,9 @@ class PartList(generics.ListCreateAPIView): except PartCategory.DoesNotExist: pass + # Ensure that related models are pre-loaded to reduce DB trips + parts_list = self.get_serializer_class().setup_eager_loading(parts_list) + return parts_list permission_classes = [ @@ -200,10 +203,14 @@ class BomList(generics.ListCreateAPIView): - GET: Return list of BomItem objects - POST: Create a new BomItem object """ - - queryset = BomItem.objects.all() + serializer_class = BomItemSerializer + def get_queryset(self): + queryset = BomItem.objects.all() + queryset = self.get_serializer_class().setup_eager_loading(queryset) + return queryset + permission_classes = [ permissions.IsAuthenticatedOrReadOnly, ] diff --git a/InvenTree/part/migrations/0001_initial.py b/InvenTree/part/migrations/0001_initial.py index c0413a909a..e5183e2b5c 100644 --- a/InvenTree/part/migrations/0001_initial.py +++ b/InvenTree/part/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 12:04 import InvenTree.validators from django.conf import settings diff --git a/InvenTree/part/migrations/0002_auto_20190519_0004.py b/InvenTree/part/migrations/0002_auto_20190520_2204.py similarity index 97% rename from InvenTree/part/migrations/0002_auto_20190519_0004.py rename to InvenTree/part/migrations/0002_auto_20190520_2204.py index 02df1a7fad..cf968b43cd 100644 --- a/InvenTree/part/migrations/0002_auto_20190519_0004.py +++ b/InvenTree/part/migrations/0002_auto_20190520_2204.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2019-05-18 14:04 +# Generated by Django 2.2 on 2019-05-20 12:04 from django.conf import settings from django.db import migrations, models @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('company', '0002_auto_20190519_0004'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('stock', '0001_initial'), ('part', '0001_initial'), + ('company', '0002_auto_20190520_2204'), + ('stock', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index d8b7529b23..04277508c7 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -16,6 +16,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.db import models, transaction +from django.db.models import Sum from django.core.validators import MinValueValidator from django.contrib.staticfiles.templatetags.staticfiles import static @@ -410,7 +411,7 @@ class Part(models.Model): total = None # Calculate the minimum number of parts that can be built using each sub-part - for item in self.bom_items.all(): + for item in self.bom_items.all().select_related('sub_part'): stock = item.sub_part.available_stock n = int(1.0 * stock / item.quantity) @@ -448,7 +449,7 @@ class Part(models.Model): builds = [] - for item in self.used_in.all(): + for item in self.used_in.all().prefetch_related('part'): for build in item.part.active_builds: b = {} @@ -462,7 +463,7 @@ class Part(models.Model): @property def allocated_build_count(self): - """ Return the total number of this that are allocated for builds + """ Return the total number of this part that are allocated for builds """ return sum([a['quantity'] for a in self.build_allocation]) @@ -481,7 +482,13 @@ class Part(models.Model): @property def stock_entries(self): - return [loc for loc in self.locations.all() if loc.in_stock] + """ Return all 'in stock' items. To be in stock: + + - customer is None + - belongs_to is None + """ + + return self.stock_items.filter(customer=None, belongs_to=None) @property def total_stock(self): @@ -489,7 +496,12 @@ class Part(models.Model): Part may be stored in multiple locations """ - return sum([loc.quantity for loc in self.stock_entries]) + total = self.stock_entries.aggregate(total=Sum('quantity'))['total'] + + if total: + return total + else: + return 0 @property def has_bom(self): @@ -518,9 +530,9 @@ class Part(models.Model): returns a string representation of a hash object which can be compared with a stored value """ - hash = hashlib.md5('bom seed'.encode()) + hash = hashlib.md5(str(self.id).encode()) - for item in self.bom_items.all(): + for item in self.bom_items.all().prefetch_related('sub_part'): hash.update(str(item.sub_part.id).encode()) hash.update(str(item.sub_part.full_name).encode()) hash.update(str(item.quantity).encode()) @@ -552,7 +564,7 @@ class Part(models.Model): def required_parts(self): """ Return a list of parts required to make this part (list of BOM items) """ parts = [] - for bom in self.bom_items.all(): + for bom in self.bom_items.all().select_related('sub_part'): parts.append(bom.sub_part) return parts @@ -561,43 +573,21 @@ class Part(models.Model): """ Return the number of supplier parts available for this part """ return self.supplier_parts.count() - @property - def min_single_price(self): - return self.get_min_supplier_price(1) - - @property - def max_single_price(self): - return self.get_max_supplier_price(1) - - @property - def min_bom_price(self): - return self.get_min_bom_price(1) - - @property - def max_bom_price(self): - return self.get_max_bom_price(1) - @property def has_pricing_info(self): """ Return true if there is pricing information for this part """ - return self.get_min_price() is not None + return self.get_price_range() is not None @property def has_complete_bom_pricing(self): """ Return true if there is pricing information for each item in the BOM. """ - for item in self.bom_items.all(): + for item in self.bom_items.all().select_related('sub_part'): if not item.sub_part.has_pricing_info: return False return True - @property - def single_price_info(self): - """ Return a simplified pricing string for this part at single quantity """ - - return self.get_price_info() - def get_price_info(self, quantity=1, buy=True, bom=True): """ Return a simplified pricing string for this part @@ -606,72 +596,44 @@ class Part(models.Model): buy: Include supplier pricing (default = True) bom: Include BOM pricing (default = True) """ - - min_price = self.get_min_price(quantity, buy, bom) - max_price = self.get_max_price(quantity, buy, bom) - if min_price is None: + price_range = self.get_price_range(quantity, buy, bom) + + if price_range is None: return None + min_price, max_price = price_range + if min_price == max_price: return min_price - return "{a} to {b}".format(a=min_price, b=max_price) + return "{a} - {b}".format(a=min_price, b=max_price) - def get_min_supplier_price(self, quantity=1): - """ Return the minimum price of this part from all available suppliers. + def get_supplier_price_range(self, quantity=1): - Args: - quantity: Number of units we wish to purchase (default = 1) - - Returns: - Numerical price if pricing is available, else None - """ - min_price = None - - for supplier_part in self.supplier_parts.all(): - supplier_price = supplier_part.get_price(quantity) - - if supplier_price is None: - continue - - if min_price is None or supplier_price < min_price: - min_price = supplier_price - - if min_price is None: - return None - else: - return min_price - - def get_max_supplier_price(self, quantity=1): - """ Return the maximum price of this part from all available suppliers. - - Args: - quantity: Number of units we wish to purchase (default = 1) - - Returns: - Numerical price if pricing is available, else None - """ - max_price = None - for supplier_part in self.supplier_parts.all(): - supplier_price = supplier_part.get_price(quantity) + for supplier in self.supplier_parts.all(): - if supplier_price is None: + price = supplier.get_price(quantity) + + if price is None: continue - if max_price is None or supplier_price > max_price: - max_price = supplier_price + if min_price is None or price < min_price: + min_price = price - if max_price is None: + if max_price is None or price > max_price: + max_price = price + + if min_price is None or max_price is None: return None - else: - return max_price - def get_min_bom_price(self, quantity=1): - """ Return the minimum price of the BOM for this part. + return (min_price, max_price) + + def get_bom_price_range(self, quantity=1): + """ Return the price range of the BOM for this part. Adds the minimum price for all components in the BOM. Note: If the BOM contains items without pricing information, @@ -679,45 +641,33 @@ class Part(models.Model): """ min_price = None + max_price = None - for item in self.bom_items.all(): - price = item.sub_part.get_min_price(quantity * item.quantity) + for item in self.bom_items.all().select_related('sub_part'): + prices = item.sub_part.get_price_range(quantity * item.quantity) - if price is None: + if prices is None: continue + low, high = prices + if min_price is None: min_price = 0 - min_price += price - - return min_price - - def get_max_bom_price(self, quantity=1): - """ Return the maximum price of the BOM for this part. - Adds the maximum price for all components in the BOM. - - Note: If the BOM contains items without pricing information, - these items cannot be included in the BOM! - """ - - max_price = None - - for item in self.bom_items.all(): - price = item.sub_part.get_max_price(quantity * item.quantity) - - if price is None: - continue - if max_price is None: max_price = 0 - max_price += price + min_price += low + max_price += high - return max_price + if min_price is None or max_price is None: + return None - def get_min_price(self, quantity=1, buy=True, bom=True): - """ Return the minimum price for this part. This price can be either: + return (min_price, max_price) + + def get_price_range(self, quantity=1, buy=True, bom=True): + + """ Return the price range for this part. This price can be either: - Supplier price (if purchased from suppliers) - BOM price (if built from other parts) @@ -726,37 +676,20 @@ class Part(models.Model): Minimum of the supplier price or BOM price. If no pricing available, returns None """ - buy_price = self.get_min_supplier_price(quantity) if buy else None - bom_price = self.get_min_bom_price(quantity) if bom else None + buy_price_range = self.get_supplier_price_range(quantity) if buy else None + bom_price_range = self.get_bom_price_range(quantity) if bom else None - if buy_price is None: - return bom_price + if buy_price_range is None: + return bom_price_range - if bom_price is None: - return buy_price - - return min(buy_price, bom_price) + elif bom_price_range is None: + return buy_price_range - def get_max_price(self, quantity=1, buy=True, bom=True): - """ Return the maximum price for this part. This price can be either: - - - Supplier price (if purchsed from suppliers) - - BOM price (if built from other parts) - - Returns: - Maximum of the supplier price or BOM price. If no pricing available, returns None - """ - - buy_price = self.get_max_supplier_price(quantity) if buy else None - bom_price = self.get_max_bom_price(quantity) if bom else None - - if buy_price is None: - return bom_price - - if bom_price is None: - return buy_price - - return max(buy_price, bom_price) + else: + return ( + min(buy_price_range[0], bom_price_range[0]), + max(buy_price_range[1], bom_price_range[1]) + ) def deepCopy(self, other, **kwargs): """ Duplicates non-field data from another part. @@ -995,8 +928,3 @@ class BomItem(models.Model): base_quantity = self.quantity * build_quantity return base_quantity + self.get_overage_quantity(base_quantity) - - @property - def price_info(self): - """ Return the price for this item in the BOM """ - return self.sub_part.get_price_info(self.quantity) diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 593f7003f9..ddbbab9e57 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -34,7 +34,14 @@ class PartBriefSerializer(serializers.ModelSerializer): url = serializers.CharField(source='get_absolute_url', read_only=True) image_url = serializers.CharField(source='get_image_url', read_only=True) - single_price_info = serializers.CharField(read_only=True) + + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('category') + queryset = queryset.prefetch_related('stock_items') + queryset = queryset.prefetch_related('bom_items') + queryset = queryset.prefetch_related('builds') + return queryset class Meta: model = Part @@ -44,7 +51,6 @@ class PartBriefSerializer(serializers.ModelSerializer): 'full_name', 'description', 'available_stock', - 'single_price_info', 'image_url', ] @@ -58,6 +64,14 @@ class PartSerializer(serializers.ModelSerializer): image_url = serializers.CharField(source='get_image_url', read_only=True) category_name = serializers.CharField(source='category_path', read_only=True) + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('category') + queryset = queryset.prefetch_related('stock_items') + queryset = queryset.prefetch_related('bom_items') + queryset = queryset.prefetch_related('builds') + return queryset + class Meta: model = Part partial = True @@ -108,7 +122,16 @@ class BomItemSerializer(InvenTreeModelSerializer): part_detail = PartBriefSerializer(source='part', many=False, read_only=True) sub_part_detail = PartBriefSerializer(source='sub_part', many=False, read_only=True) - price_info = serializers.CharField(read_only=True) + + @staticmethod + def setup_eager_loading(queryset): + queryset = queryset.prefetch_related('part') + queryset = queryset.prefetch_related('part__category') + queryset = queryset.prefetch_related('part__stock_items') + queryset = queryset.prefetch_related('sub_part') + queryset = queryset.prefetch_related('sub_part__category') + queryset = queryset.prefetch_related('sub_part__stock_items') + return queryset class Meta: model = BomItem @@ -119,7 +142,6 @@ class BomItemSerializer(InvenTreeModelSerializer): 'sub_part', 'sub_part_detail', 'quantity', - 'price_info', 'overage', 'note', ] diff --git a/InvenTree/part/templates/part/bom.html b/InvenTree/part/templates/part/bom.html index bff082b6ae..19b876d905 100644 --- a/InvenTree/part/templates/part/bom.html +++ b/InvenTree/part/templates/part/bom.html @@ -16,9 +16,6 @@ The BOM for {{ part.full_name }} does not have complete pricing information {% endif %} -