From a9e22d0ae9d3dd992f3bb86bc02424c4e1419f06 Mon Sep 17 00:00:00 2001 From: luwol03 Date: Fri, 5 Aug 2022 00:07:12 +0200 Subject: [PATCH] Fix: Treegrid is loading an eternity for huge amounts of data (#3451) * Added default max depth and lazy loading to StorageLocation * Added default max depth and lazy loading to PartCategory * Update API version * lint: fix * Added INVENTREE_TREE_DEPTH setting * Refactored int conversion into own helper function * Added tests --- InvenTree/InvenTree/api_version.py | 6 +- InvenTree/InvenTree/helpers.py | 16 +++++ InvenTree/common/models.py | 10 +++ InvenTree/part/api.py | 11 +++- InvenTree/part/test_api.py | 45 ++++++-------- InvenTree/stock/api.py | 10 ++- InvenTree/stock/test_api.py | 43 +++++++++++-- .../templates/InvenTree/settings/global.html | 1 + InvenTree/templates/js/translated/part.js | 58 +++++++++++++++++- InvenTree/templates/js/translated/stock.js | 61 ++++++++++++++++++- 10 files changed, 225 insertions(+), 36 deletions(-) diff --git a/InvenTree/InvenTree/api_version.py b/InvenTree/InvenTree/api_version.py index 86381c70ee..f68a04cfa3 100644 --- a/InvenTree/InvenTree/api_version.py +++ b/InvenTree/InvenTree/api_version.py @@ -2,11 +2,15 @@ # InvenTree API version -INVENTREE_API_VERSION = 69 +INVENTREE_API_VERSION = 70 """ Increment this API version number whenever there is a significant change to the API that any clients need to know about +v70 -> 2022-08-02 : https://github.com/inventree/InvenTree/pull/3451 + - Adds a 'depth' parameter to the PartCategory list API + - Adds a 'depth' parameter to the StockLocation list API + v69 -> 2022-08-01 : https://github.com/inventree/InvenTree/pull/3443 - Updates the PartCategory list API: - Improve query efficiency: O(n) becomes O(1) diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 7ad1781e61..046f3f2d5c 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -283,6 +283,22 @@ def str2bool(text, test=True): return str(text).lower() in ['0', 'n', 'no', 'none', 'f', 'false', 'off', ] +def str2int(text, default=None): + """Convert a string to int if possible + + Args: + text: Int like string + default: Return value if str is no int like + + Returns: + Converted int value + """ + try: + return int(text) + except Exception: + return default + + def is_bool(text): """Determine if a string value 'looks' like a boolean.""" if str2bool(text, True): diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 1c9817b262..a006dab47b 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -876,6 +876,16 @@ class InvenTreeSetting(BaseInvenTreeSetting): 'default': True, }, + 'INVENTREE_TREE_DEPTH': { + 'name': _('Tree Depth'), + 'description': _('Default tree depth for treeview. Deeper levels can be lazy loaded as they are needed.'), + 'default': 1, + 'validator': [ + int, + MinValueValidator(0), + ] + }, + 'BARCODE_ENABLE': { 'name': _('Barcode Support'), 'description': _('Enable barcode scanner support'), diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index d95a93815b..1d5b10ac61 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -25,7 +25,8 @@ from company.models import Company, ManufacturerPart, SupplierPart from InvenTree.api import (APIDownloadMixin, AttachmentMixin, ListCreateDestroyAPIView) from InvenTree.filters import InvenTreeOrderingFilter -from InvenTree.helpers import DownloadFile, increment, isNull, str2bool +from InvenTree.helpers import (DownloadFile, increment, isNull, str2bool, + str2int) from InvenTree.mixins import (CreateAPI, ListAPI, ListCreateAPI, RetrieveAPI, RetrieveUpdateAPI, RetrieveUpdateDestroyAPI, UpdateAPI) @@ -85,6 +86,8 @@ class CategoryList(ListCreateAPI): cascade = str2bool(params.get('cascade', False)) + depth = str2int(params.get('depth', None)) + # Do not filter by category if cat_id is None: pass @@ -94,12 +97,18 @@ class CategoryList(ListCreateAPI): if not cascade: queryset = queryset.filter(parent=None) + if cascade and depth is not None: + queryset = queryset.filter(level__lte=depth) + else: try: category = PartCategory.objects.get(pk=cat_id) if cascade: parents = category.get_descendants(include_self=True) + if depth is not None: + parents = parents.filter(level__lte=category.level + depth) + parent_ids = [p.id for p in parents] queryset = queryset.filter(parent__in=parent_ids) diff --git a/InvenTree/part/test_api.py b/InvenTree/part/test_api.py index 7c4bdf67a1..733c97abe4 100644 --- a/InvenTree/part/test_api.py +++ b/InvenTree/part/test_api.py @@ -49,33 +49,25 @@ class PartCategoryAPITest(InvenTreeAPITestCase): """Test the PartCategoryList API endpoint""" url = reverse('api-part-category-list') - response = self.get(url, expected_code=200) + test_cases = [ + ({}, 8, 'no parameters'), + ({'parent': 1, 'cascade': False}, 3, 'Filter by parent, no cascading'), + ({'parent': 1, 'cascade': True}, 5, 'Filter by parent, cascading'), + ({'cascade': True, 'depth': 0}, 8, 'Cascade with no parent, depth=0'), + ({'cascade': False, 'depth': 10}, 8, 'Cascade with no parent, depth=0'), + ({'parent': 'null', 'cascade': True, 'depth': 0}, 2, 'Cascade with null parent, depth=0'), + ({'parent': 'null', 'cascade': True, 'depth': 10}, 8, 'Cascade with null parent and bigger depth'), + ({'parent': 'null', 'cascade': False, 'depth': 10}, 2, 'No cascade even with depth specified with null parent'), + ({'parent': 1, 'cascade': False, 'depth': 0}, 3, 'Dont cascade with depth=0 and parent'), + ({'parent': 1, 'cascade': True, 'depth': 0}, 3, 'Cascade with depth=0 and parent'), + ({'parent': 1, 'cascade': False, 'depth': 1}, 3, 'Dont cascade even with depth=1 specified with parent'), + ({'parent': 1, 'cascade': True, 'depth': 1}, 5, 'Cascade with depth=1 with parent'), + ({'parent': 1, 'cascade': True, 'depth': 'abcdefg'}, 5, 'Cascade with invalid depth and parent'), + ] - self.assertEqual(len(response.data), 8) - - # Filter by parent, depth=1 - response = self.get( - url, - { - 'parent': 1, - 'cascade': False, - }, - expected_code=200 - ) - - self.assertEqual(len(response.data), 3) - - # Filter by parent, cascading - response = self.get( - url, - { - 'parent': 1, - 'cascade': True, - }, - expected_code=200, - ) - - self.assertEqual(len(response.data), 5) + for params, res_len, description in test_cases: + response = self.get(url, params, expected_code=200) + self.assertEqual(len(response.data), res_len, description) # Check that the required fields are present fields = [ @@ -90,6 +82,7 @@ class PartCategoryAPITest(InvenTreeAPITestCase): 'url' ] + response = self.get(url, expected_code=200) for result in response.data: for f in fields: self.assertIn(f, result) diff --git a/InvenTree/stock/api.py b/InvenTree/stock/api.py index a279f71d2a..7ecf3ee480 100644 --- a/InvenTree/stock/api.py +++ b/InvenTree/stock/api.py @@ -26,7 +26,7 @@ from InvenTree.api import (APIDownloadMixin, AttachmentMixin, ListCreateDestroyAPIView) from InvenTree.filters import InvenTreeOrderingFilter from InvenTree.helpers import (DownloadFile, extract_serial_numbers, isNull, - str2bool) + str2bool, str2int) from InvenTree.mixins import (CreateAPI, ListAPI, ListCreateAPI, RetrieveAPI, RetrieveUpdateAPI, RetrieveUpdateDestroyAPI) from order.models import PurchaseOrder, SalesOrder, SalesOrderAllocation @@ -241,6 +241,8 @@ class StockLocationList(ListCreateAPI): cascade = str2bool(params.get('cascade', False)) + depth = str2int(params.get('depth', None)) + # Do not filter by location if loc_id is None: pass @@ -251,6 +253,9 @@ class StockLocationList(ListCreateAPI): if not cascade: queryset = queryset.filter(parent=None) + if cascade and depth is not None: + queryset = queryset.filter(level__lte=depth) + else: try: @@ -259,6 +264,9 @@ class StockLocationList(ListCreateAPI): # All sub-locations to be returned too? if cascade: parents = location.get_descendants(include_self=True) + if depth is not None: + parents = parents.filter(level__lte=location.level + depth) + parent_ids = [p.id for p in parents] queryset = queryset.filter(parent__in=parent_ids) diff --git a/InvenTree/stock/test_api.py b/InvenTree/stock/test_api.py index e35eabc7df..a9deeee1f5 100644 --- a/InvenTree/stock/test_api.py +++ b/InvenTree/stock/test_api.py @@ -54,11 +54,44 @@ class StockLocationTest(StockAPITestCase): StockLocation.objects.create(name='top', description='top category') def test_list(self): - """Test StockLocation list.""" - # Check that we can request the StockLocation list - response = self.client.get(self.list_url, format='json') - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertGreaterEqual(len(response.data), 1) + """Test the StockLocationList API endpoint""" + test_cases = [ + ({}, 8, 'no parameters'), + ({'parent': 1, 'cascade': False}, 2, 'Filter by parent, no cascading'), + ({'parent': 1, 'cascade': True}, 2, 'Filter by parent, cascading'), + ({'cascade': True, 'depth': 0}, 8, 'Cascade with no parent, depth=0'), + ({'cascade': False, 'depth': 10}, 8, 'Cascade with no parent, depth=0'), + ({'parent': 'null', 'cascade': True, 'depth': 0}, 7, 'Cascade with null parent, depth=0'), + ({'parent': 'null', 'cascade': True, 'depth': 10}, 8, 'Cascade with null parent and bigger depth'), + ({'parent': 'null', 'cascade': False, 'depth': 10}, 3, 'No cascade even with depth specified with null parent'), + ({'parent': 1, 'cascade': False, 'depth': 0}, 2, 'Dont cascade with depth=0 and parent'), + ({'parent': 1, 'cascade': True, 'depth': 0}, 2, 'Cascade with depth=0 and parent'), + ({'parent': 1, 'cascade': False, 'depth': 1}, 2, 'Dont cascade even with depth=1 specified with parent'), + ({'parent': 1, 'cascade': True, 'depth': 1}, 2, 'Cascade with depth=1 with parent'), + ({'parent': 1, 'cascade': True, 'depth': 'abcdefg'}, 2, 'Cascade with invalid depth and parent'), + ] + + for params, res_len, description in test_cases: + response = self.get(self.list_url, params, expected_code=200) + self.assertEqual(len(response.data), res_len, description) + + # Check that the required fields are present + fields = [ + 'pk', + 'name', + 'description', + 'level', + 'parent', + 'items', + 'pathstring', + 'owner', + 'url' + ] + + response = self.get(self.list_url, expected_code=200) + for result in response.data: + for f in fields: + self.assertIn(f, result) def test_add(self): """Test adding StockLocation.""" diff --git a/InvenTree/templates/InvenTree/settings/global.html b/InvenTree/templates/InvenTree/settings/global.html index 4b431e46f2..f6812bdc34 100644 --- a/InvenTree/templates/InvenTree/settings/global.html +++ b/InvenTree/templates/InvenTree/settings/global.html @@ -22,6 +22,7 @@ {% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_FROM_URL" icon="fa-cloud-download-alt" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE" icon="fa-server" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_REQUIRE_CONFIRM" icon="fa-check" %} + {% include "InvenTree/settings/setting.html" with key="INVENTREE_TREE_DEPTH" icon="fa-sitemap" %} diff --git a/InvenTree/templates/js/translated/part.js b/InvenTree/templates/js/translated/part.js index a5f284aabc..2b6e6f0f41 100644 --- a/InvenTree/templates/js/translated/part.js +++ b/InvenTree/templates/js/translated/part.js @@ -1750,6 +1750,7 @@ function loadPartCategoryTable(table, options) { if (tree_view) { params.cascade = true; + params.depth = global_settings.INVENTREE_TREE_DEPTH; } var original = {}; @@ -1761,6 +1762,35 @@ function loadPartCategoryTable(table, options) { setupFilterList(filterKey, table, filterListElement); + // Function to request sub-category items + function requestSubItems(parent_pk) { + inventreeGet( + options.url || '{% url "api-part-category-list" %}', + { + parent: parent_pk, + }, + { + success: function(response) { + // Add the returned sub-items to the table + for (var idx = 0; idx < response.length; idx++) { + response[idx].parent = parent_pk; + } + + const row = $(table).bootstrapTable('getRowByUniqueId', parent_pk); + row.subReceived = true; + + $(table).bootstrapTable('updateByUniqueId', parent_pk, row, true); + + table.bootstrapTable('append', response); + }, + error: function(xhr) { + console.error('Error requesting sub-category for category=' + parent_pk); + showApiError(xhr); + } + } + ); + } + table.inventreeTable({ treeEnable: tree_view, rootParentId: tree_view ? options.params.parent : null, @@ -1839,6 +1869,20 @@ function loadPartCategoryTable(table, options) { } }); + + // Callback for 'load sub category' button + $(table).find('.load-sub-category').click(function(event) { + event.preventDefault(); + + const pk = $(this).attr('pk'); + const row = $(table).bootstrapTable('getRowByUniqueId', pk); + + // Request sub-category for this category + requestSubItems(row.pk); + + row.subRequested = true; + $(table).bootstrapTable('updateByUniqueId', pk, row, true); + }); } else { $('#view-category-tree').removeClass('btn-secondary').addClass('btn-outline-secondary'); $('#view-category-list').removeClass('btn-outline-secondary').addClass('btn-secondary'); @@ -1859,8 +1903,20 @@ function loadPartCategoryTable(table, options) { switchable: true, sortable: true, formatter: function(value, row) { + let html = ''; - var html = renderLink( + if (row._level >= global_settings.INVENTREE_TREE_DEPTH && !row.subReceived) { + if (row.subRequested) { + html += ``; + } else { + html += ` + + + `; + } + } + + html += renderLink( value, `/part/category/${row.pk}/` ); diff --git a/InvenTree/templates/js/translated/stock.js b/InvenTree/templates/js/translated/stock.js index cb065f68b7..21ef8c215d 100644 --- a/InvenTree/templates/js/translated/stock.js +++ b/InvenTree/templates/js/translated/stock.js @@ -2226,6 +2226,7 @@ function loadStockLocationTable(table, options) { if (tree_view) { params.cascade = true; + params.depth = global_settings.INVENTREE_TREE_DEPTH; } var filters = {}; @@ -2248,6 +2249,35 @@ function loadStockLocationTable(table, options) { filters[key] = params[key]; } + // Function to request sub-location items + function requestSubItems(parent_pk) { + inventreeGet( + options.url || '{% url "api-location-list" %}', + { + parent: parent_pk, + }, + { + success: function(response) { + // Add the returned sub-items to the table + for (var idx = 0; idx < response.length; idx++) { + response[idx].parent = parent_pk; + } + + const row = $(table).bootstrapTable('getRowByUniqueId', parent_pk); + row.subReceived = true; + + $(table).bootstrapTable('updateByUniqueId', parent_pk, row, true); + + table.bootstrapTable('append', response); + }, + error: function(xhr) { + console.error('Error requesting sub-locations for location=' + parent_pk); + showApiError(xhr); + } + } + ); + } + table.inventreeTable({ treeEnable: tree_view, rootParentId: tree_view ? options.params.parent : null, @@ -2286,6 +2316,20 @@ function loadStockLocationTable(table, options) { } }); + + // Callback for 'load sub location' button + $(table).find('.load-sub-location').click(function(event) { + event.preventDefault(); + + const pk = $(this).attr('pk'); + const row = $(table).bootstrapTable('getRowByUniqueId', pk); + + // Request sub-location for this location + requestSubItems(row.pk); + + row.subRequested = true; + $(table).bootstrapTable('updateByUniqueId', pk, row, true); + }); } else { $('#view-location-tree').removeClass('btn-secondary').addClass('btn-outline-secondary'); $('#view-location-list').removeClass('btn-outline-secondary').addClass('btn-secondary'); @@ -2345,10 +2389,25 @@ function loadStockLocationTable(table, options) { switchable: true, sortable: true, formatter: function(value, row) { - return renderLink( + let html = ''; + + if (row._level >= global_settings.INVENTREE_TREE_DEPTH && !row.subReceived) { + if (row.subRequested) { + html += ``; + } else { + html += ` + + + `; + } + } + + html += renderLink( value, `/stock/location/${row.pk}/` ); + + return html; }, }, {