Enable regex search (#4566)

* Adds custom search filter to allow 'regex' searching of results

* Specify if "shell" can access database for certain commands

* Bug fix for settings API

- Do not allow cache on detail endpoints
- Was causing strange error conditions with missing or duplicate PK values

* Adds user setting to control regex search

* Enable regex for search queries

- bootstrap tables
- search preview

* Pass search options through bettererer

* Refactor API endpoints to use new filter approach

* Bump API version

* Add "whole word" search

- Closes https://github.com/inventree/InvenTree/issues/4510

* Handle case where existing fields are empty

* pop > get
This commit is contained in:
Oliver 2023-04-04 07:05:55 +10:00 committed by GitHub
parent eef303dfea
commit d6715d94c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 165 additions and 72 deletions

View File

@ -13,6 +13,7 @@ from rest_framework.serializers import ValidationError
from rest_framework.views import APIView
import users.models
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.mixins import ListCreateAPI
from InvenTree.permissions import RolePermission
from part.templatetags.inventree_extras import plugins_info
@ -203,8 +204,8 @@ class AttachmentMixin:
filter_backends = [
DjangoFilterBackend,
InvenTreeSearchFilter,
filters.OrderingFilter,
filters.SearchFilter,
]
def perform_create(self, serializer):
@ -255,32 +256,25 @@ class APISearchView(APIView):
data = request.data
search = data.get('search', '')
# Enforce a 'limit' parameter
try:
limit = int(data.get('limit', 1))
except ValueError:
limit = 1
try:
offset = int(data.get('offset', 0))
except ValueError:
offset = 0
results = {}
# These parameters are passed through to the individual queries, with optional default values
pass_through_params = {
'search': '',
'search_regex': False,
'search_whole': False,
'limit': 1,
'offset': 0,
}
for key, cls in self.get_result_types().items():
# Only return results which are specifically requested
if key in data:
params = data[key]
params['search'] = search
# Enforce limit
params['limit'] = limit
params['offset'] = offset
for k, v in pass_through_params.items():
params[k] = request.data.get(k, v)
# Enforce json encoding
params['format'] = 'json'

View File

@ -2,11 +2,14 @@
# InvenTree API version
INVENTREE_API_VERSION = 105
INVENTREE_API_VERSION = 106
"""
Increment this API version number whenever there is a significant change to the API that any clients need to know about
v106 -> 2023-04-03 : https://github.com/inventree/InvenTree/pull/4566
- Adds 'search_regex' parameter to all searchable API endpoints
v105 -> 2023-03-31 : https://github.com/inventree/InvenTree/pull/4543
- Adds API endpoints for status label information on various models

View File

@ -1,6 +1,62 @@
"""General filters for InvenTree."""
from rest_framework.filters import OrderingFilter
from rest_framework.filters import OrderingFilter, SearchFilter
from InvenTree.helpers import str2bool
class InvenTreeSearchFilter(SearchFilter):
"""Custom search filter which allows adjusting of search terms dynamically"""
def get_search_fields(self, view, request):
"""Return a set of search fields for the request, adjusted based on request params.
The following query params are available to 'augment' the search (in decreasing order of priority)
- search_regex: If True, search is perfomed on 'regex' comparison
"""
regex = str2bool(request.query_params.get('search_regex', False))
search_fields = super().get_search_fields(view, request)
fields = []
if search_fields:
for field in search_fields:
if regex:
field = '$' + field
fields.append(field)
return fields
def get_search_terms(self, request):
"""Return the search terms for this search request.
Depending on the request parameters, we may "augment" these somewhat
"""
whole = str2bool(request.query_params.get('search_whole', False))
terms = []
search_terms = super().get_search_terms(request)
if search_terms:
for term in search_terms:
term = term.strip()
if not term:
# Ignore blank inputs
continue
if whole:
# Wrap the search term to enable word-boundary matching
term = r"\y" + term + r"\y"
terms.append(term)
return terms
class InvenTreeOrderingFilter(OrderingFilter):

View File

@ -13,7 +13,7 @@ def isImportingData():
return 'loaddata' in sys.argv
def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False):
def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False, allow_shell: bool = False):
"""Returns True if the apps.py file can access database records.
There are some circumstances where we don't want the ready function in apps.py
@ -26,7 +26,6 @@ def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False):
'loaddata',
'dumpdata',
'check',
'shell',
'createsuperuser',
'wait_for_db',
'prerender',
@ -42,6 +41,9 @@ def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False):
'mediarestore',
]
if not allow_shell:
excluded_commands.append('shell')
if not allow_test:
# Override for testing mode?
excluded_commands.append('test')

View File

@ -4,7 +4,6 @@ from django.urls import include, path, re_path
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.models import User
from rest_framework import filters
from rest_framework.exceptions import ValidationError
from django_filters.rest_framework import DjangoFilterBackend
@ -12,7 +11,7 @@ from django_filters import rest_framework as rest_filters
from InvenTree.api import AttachmentMixin, APIDownloadMixin, ListCreateDestroyAPIView, MetadataView, StatusView
from InvenTree.helpers import str2bool, isNull, DownloadFile
from InvenTree.filters import InvenTreeOrderingFilter
from InvenTree.filters import InvenTreeOrderingFilter, InvenTreeSearchFilter
from InvenTree.status_codes import BuildStatus
from InvenTree.mixins import CreateAPI, RetrieveUpdateDestroyAPI, ListCreateAPI
@ -101,7 +100,7 @@ class BuildList(APIDownloadMixin, ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]

View File

@ -20,6 +20,7 @@ import common.models
import common.serializers
from InvenTree.api import BulkDeleteMixin
from InvenTree.config import CONFIG_LOOKUPS
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.helpers import inheritors
from InvenTree.mixins import (ListAPI, RetrieveAPI, RetrieveUpdateAPI,
RetrieveUpdateDestroyAPI)
@ -169,7 +170,7 @@ class SettingsList(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -226,7 +227,10 @@ class GlobalSettingsDetail(RetrieveUpdateAPI):
if key not in common.models.InvenTreeSetting.SETTINGS.keys():
raise NotFound()
return common.models.InvenTreeSetting.get_setting_object(key)
return common.models.InvenTreeSetting.get_setting_object(
key,
cache=False, create=True
)
permission_classes = [
permissions.IsAuthenticated,
@ -284,7 +288,11 @@ class UserSettingsDetail(RetrieveUpdateAPI):
if key not in common.models.InvenTreeUserSetting.SETTINGS.keys():
raise NotFound()
return common.models.InvenTreeUserSetting.get_setting_object(key, user=self.request.user)
return common.models.InvenTreeUserSetting.get_setting_object(
key,
user=self.request.user,
cache=False, create=True
)
permission_classes = [
UserSettingsPermissions,
@ -334,7 +342,7 @@ class NotificationList(NotificationMessageMixin, BulkDeleteMixin, ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]

View File

@ -388,10 +388,10 @@ class BaseInvenTreeSetting(models.Model):
if not setting:
# Unless otherwise specified, attempt to create the setting
create = kwargs.get('create', True)
create = kwargs.pop('create', True)
# Prevent creation of new settings objects when importing data
if InvenTree.ready.isImportingData() or not InvenTree.ready.canAppAccessDatabase(allow_test=True):
if InvenTree.ready.isImportingData() or not InvenTree.ready.canAppAccessDatabase(allow_test=True, allow_shell=True):
create = False
if create:
@ -1979,6 +1979,20 @@ class InvenTreeUserSetting(BaseInvenTreeSetting):
'validator': [int, MinValueValidator(1)]
},
'SEARCH_REGEX': {
'name': _('Regex Search'),
'description': _('Enable regular expressions in search queries'),
'default': False,
'validator': bool,
},
'SEARCH_WHOLE': {
'name': _('Whole Word Search'),
'description': _('Search queries return results for whole word matches'),
'default': False,
'validator': bool,
},
'PART_SHOW_QUANTITY_IN_FORMS': {
'name': _('Show Quantity in Forms'),
'description': _('Display available part quantity in some forms'),

View File

@ -10,7 +10,7 @@ from rest_framework import filters
import part.models
from InvenTree.api import (AttachmentMixin, ListCreateDestroyAPIView,
MetadataView)
from InvenTree.filters import InvenTreeOrderingFilter
from InvenTree.filters import InvenTreeOrderingFilter, InvenTreeSearchFilter
from InvenTree.helpers import str2bool
from InvenTree.mixins import ListCreateAPI, RetrieveUpdateDestroyAPI
@ -46,7 +46,7 @@ class CompanyList(ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -116,7 +116,7 @@ class ContactList(ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -194,7 +194,7 @@ class ManufacturerPartList(ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -290,7 +290,7 @@ class ManufacturerPartParameterList(ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -398,7 +398,7 @@ class SupplierPartList(ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]

View File

@ -8,12 +8,12 @@ from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page, never_cache
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters
from rest_framework.exceptions import NotFound
import common.models
import InvenTree.helpers
from InvenTree.api import MetadataView
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.mixins import ListAPI, RetrieveAPI, RetrieveUpdateDestroyAPI
from InvenTree.tasks import offload_task
from part.models import Part
@ -124,7 +124,7 @@ class LabelListView(LabelFilterMixin, ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter
InvenTreeSearchFilter
]
filterset_fields = [

View File

@ -21,7 +21,7 @@ from common.settings import settings
from company.models import SupplierPart
from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
ListCreateDestroyAPIView, MetadataView, StatusView)
from InvenTree.filters import InvenTreeOrderingFilter
from InvenTree.filters import InvenTreeOrderingFilter, InvenTreeSearchFilter
from InvenTree.helpers import DownloadFile, str2bool
from InvenTree.mixins import (CreateAPI, ListAPI, ListCreateAPI,
RetrieveUpdateDestroyAPI)
@ -63,7 +63,7 @@ class GeneralExtraLineList(APIDownloadMixin):
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter
]
@ -309,7 +309,7 @@ class PurchaseOrderList(PurchaseOrderMixin, APIDownloadMixin, ListCreateAPI):
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -510,7 +510,7 @@ class PurchaseOrderLineItemList(PurchaseOrderLineItemMixin, APIDownloadMixin, Li
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter
]
@ -695,7 +695,7 @@ class SalesOrderList(SalesOrderMixin, APIDownloadMixin, ListCreateAPI):
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -819,7 +819,7 @@ class SalesOrderLineItemList(SalesOrderLineItemMixin, APIDownloadMixin, ListCrea
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter
]
@ -1156,7 +1156,7 @@ class ReturnOrderList(ReturnOrderMixin, APIDownloadMixin, ListCreateAPI):
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -1304,7 +1304,7 @@ class ReturnOrderLineItemList(ReturnOrderLineItemMixin, APIDownloadMixin, ListCr
filter_backends = [
rest_filters.DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]

View File

@ -17,7 +17,7 @@ import order.models
from build.models import Build, BuildItem
from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
ListCreateDestroyAPIView, MetadataView)
from InvenTree.filters import InvenTreeOrderingFilter
from InvenTree.filters import InvenTreeOrderingFilter, InvenTreeSearchFilter
from InvenTree.helpers import (DownloadFile, increment_serial_number, isNull,
str2bool, str2int)
from InvenTree.mixins import (CreateAPI, CustomRetrieveUpdateDestroyAPI,
@ -154,7 +154,7 @@ class CategoryList(CategoryMixin, APIDownloadMixin, ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -387,7 +387,7 @@ class PartTestTemplateList(ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.OrderingFilter,
filters.SearchFilter,
InvenTreeSearchFilter,
]
@ -421,7 +421,7 @@ class PartThumbs(ListAPI):
return Response(data)
filter_backends = [
filters.SearchFilter,
InvenTreeSearchFilter,
]
search_fields = [
@ -1226,7 +1226,7 @@ class PartList(PartMixin, APIDownloadMixin, ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -1340,7 +1340,7 @@ class PartParameterTemplateList(ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.OrderingFilter,
filters.SearchFilter,
InvenTreeSearchFilter,
]
filterset_fields = [
@ -1733,7 +1733,7 @@ class BomList(BomMixin, ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -1838,7 +1838,7 @@ class BomItemSubstituteList(ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]

View File

@ -328,13 +328,14 @@ def setting_object(key, *args, **kwargs):
return PluginSetting.get_setting_object(key, plugin=plugin, cache=cache)
if 'method' in kwargs:
elif 'method' in kwargs:
return NotificationUserSetting.get_setting_object(key, user=kwargs['user'], method=kwargs['method'], cache=cache)
if 'user' in kwargs:
elif 'user' in kwargs:
return InvenTreeUserSetting.get_setting_object(key, user=kwargs['user'], cache=cache)
return InvenTreeSetting.get_setting_object(key, cache=cache)
else:
return InvenTreeSetting.get_setting_object(key, cache=cache)
@register.simple_tag()

View File

@ -9,6 +9,7 @@ from rest_framework.response import Response
import plugin.serializers as PluginSerializers
from common.api import GlobalSettingsPermissions
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.mixins import (CreateAPI, ListAPI, RetrieveUpdateAPI,
RetrieveUpdateDestroyAPI, UpdateAPI)
from InvenTree.permissions import IsSuperuser
@ -58,7 +59,7 @@ class PluginList(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]

View File

@ -10,7 +10,6 @@ from django.utils.translation import gettext_lazy as _
from django.views.decorators.cache import cache_page, never_cache
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters
from rest_framework.response import Response
import build.models
@ -19,6 +18,7 @@ import InvenTree.helpers
import order.models
import part.models
from InvenTree.api import MetadataView
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.mixins import ListAPI, RetrieveAPI, RetrieveUpdateDestroyAPI
from stock.models import StockItem, StockItemAttachment
@ -35,7 +35,7 @@ class ReportListView(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
]
filterset_fields = [

View File

@ -24,7 +24,7 @@ from company.models import Company, SupplierPart
from company.serializers import CompanySerializer, SupplierPartSerializer
from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
ListCreateDestroyAPIView, MetadataView, StatusView)
from InvenTree.filters import InvenTreeOrderingFilter
from InvenTree.filters import InvenTreeOrderingFilter, InvenTreeSearchFilter
from InvenTree.helpers import (DownloadFile, extract_serial_numbers, isNull,
str2bool, str2int)
from InvenTree.mixins import (CreateAPI, CustomRetrieveUpdateDestroyAPI,
@ -296,7 +296,7 @@ class StockLocationList(APIDownloadMixin, ListCreateAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -1009,7 +1009,7 @@ class StockList(APIDownloadMixin, ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
InvenTreeOrderingFilter,
]
@ -1057,7 +1057,7 @@ class StockAttachmentList(AttachmentMixin, ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.OrderingFilter,
filters.SearchFilter,
InvenTreeSearchFilter,
]
filterset_fields = [
@ -1087,7 +1087,7 @@ class StockItemTestResultList(ListCreateDestroyAPIView):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]
@ -1312,7 +1312,7 @@ class StockTrackingList(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
filters.OrderingFilter,
]

View File

@ -14,6 +14,9 @@
<div class='row'>
<table class='table table-striped table-condensed'>
<tbody>
{% include "InvenTree/settings/setting.html" with key="SEARCH_WHOLE" user_setting=True icon='fa-spell-check' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_REGEX" user_setting=True icon='fa-code' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_RESULTS" user_setting=True icon='fa-search' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_SHOW_PARTS" user_setting=True icon='fa-shapes' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_HIDE_INACTIVE_PARTS" user_setting=True icon='fa-eye-slash' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_SHOW_SUPPLIER_PARTS" user_setting=True icon='fa-building' %}
@ -31,7 +34,6 @@
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_SHOW_RETURN_ORDERS" user_setting=True icon='fa-truck' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_EXCLUDE_INACTIVE_RETURN_ORDERS" user_setting=True icon='fa-eye-slash' %}
{% include "InvenTree/settings/setting.html" with key="SEARCH_PREVIEW_RESULTS" user_setting=True icon='fa-search' %}
</tbody>
</table>

View File

@ -141,6 +141,8 @@ function updateSearch() {
// Construct base query
searchQuery = {
search: searchTextCurrent,
search_regex: user_settings.SEARCH_REGEX ? true : false,
search_whole: user_settings.SEARCH_WHOLE ? true : false,
limit: user_settings.SEARCH_PREVIEW_RESULTS,
offset: 0,
};

View File

@ -355,6 +355,16 @@ function convertQueryParameters(params, filters) {
delete params['original_search'];
}
// Enable regex search
if (user_settings.SEARCH_REGEX) {
params['search_regex'] = true;
}
// Enable whole word search
if (user_settings.SEARCH_WHOLE) {
params['search_whole'] = true;
}
return params;
}

View File

@ -5,11 +5,12 @@ from django.core.exceptions import ObjectDoesNotExist
from django.urls import include, path, re_path
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters, permissions, status
from rest_framework import permissions, status
from rest_framework.authtoken.models import Token
from rest_framework.response import Response
from rest_framework.views import APIView
from InvenTree.filters import InvenTreeSearchFilter
from InvenTree.mixins import ListAPI, RetrieveAPI, RetrieveUpdateAPI
from InvenTree.serializers import UserSerializer
from users.models import Owner, RuleSet, check_user_role
@ -137,7 +138,7 @@ class UserList(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
]
search_fields = [
@ -168,7 +169,7 @@ class GroupList(ListAPI):
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
InvenTreeSearchFilter,
]
search_fields = [