Remove hidden characters from form fields (#3597)

* Remove control and non-printable characters from form fields (server side)

* Update regex to properly filter out control characters only

* Add regex lib to requirements flie

* Fix regex in javascript (client side)

* add required unicode flag

InvenTree/InvenTree/mixins.py

@@ -2,6 +2,7 @@
 
 from django.utils.translation import gettext_lazy as _
 
+import regex
 from bleach import clean
 from rest_framework import generics, status
 from rest_framework.exceptions import ValidationError
@@ -71,6 +72,12 @@ class CleanMixin():
                 field: [_("Remove HTML tags from this value")]
             })
 
+        # Remove ASCII control characters
+        cleaned = regex.sub(u'[\x01-\x1F]+', '', cleaned)
+
+        # Remove Unicode control characters
+        cleaned = regex.sub(u'[^\P{C}]+', '', cleaned)
+
         return cleaned
 
     def clean_data(self, data: dict) -> dict:

InvenTree/part/test_api.py

@@ -223,7 +223,10 @@ class PartCategoryAPITest(InvenTreeAPITestCase):
         self.assertEqual(PartCategoryParameterTemplate.objects.count(), 0)
 
     def test_bleach(self):
-        """Test that the data cleaning functionality is working"""
+        """Test that the data cleaning functionality is working.
+
+        This helps to protect against XSS injection
+        """
 
         url = reverse('api-part-category-detail', kwargs={'pk': 1})
 
@@ -244,6 +247,8 @@ class PartCategoryAPITest(InvenTreeAPITestCase):
                 expected_code=400
             )
 
+            self.assertIn('Remove HTML tags', str(response.data))
+
         # Raw characters should be allowed
         allowed = [
             '<< hello',
@@ -262,6 +267,30 @@ class PartCategoryAPITest(InvenTreeAPITestCase):
 
             self.assertEqual(response.data['description'], val)
 
+    def test_invisible_chars(self):
+        """Test that invisible characters are removed from the input data"""
+
+        url = reverse('api-part-category-detail', kwargs={'pk': 1})
+
+        values = [
+            'A part\n category\n\t',
+            'A\t part\t category\t',
+            'A pa\rrt cat\r\r\regory',
+            'A part\u200e catego\u200fry\u202e'
+        ]
+
+        for val in values:
+
+            response = self.patch(
+                url,
+                {
+                    'description': val,
+                },
+                expected_code=200,
+            )
+
+            self.assertEqual(response.data['description'], 'A part category')
+
 
 class PartOptionsAPITest(InvenTreeAPITestCase):
     """Tests for the various OPTIONS endpoints in the /part/ API.

InvenTree/templates/js/translated/helpers.js

@@ -341,8 +341,8 @@ function sanitizeInputString(s, options={}) {
     // Remove ASCII control characters
     s = s.replace(/[\x01-\x1F]+/g, '');
 
-    // Remove non-printable characters
+    // Remove Unicode control characters
-    s = s.replace(/[^ -~]+/g, '');
+    s = s.replace(/[\p{C}]+/gu, '');
 
     s = s.trim();
 

requirements.in

@@ -30,6 +30,7 @@ pillow                                  # Image manipulation
 python-barcode[images]                  # Barcode generator
 qrcode[pil]                             # QR code generator
 rapidfuzz==0.7.6                        # Fuzzy string matching
+regex                                   # Advanced regular expressions
 sentry-sdk                              # Error reporting (optional)
 setuptools                              # Standard depenedency
 tablib[xls,xlsx,yaml]                   # Support for XLS and XLSX formats

requirements.txt

@@ -194,6 +194,8 @@ redis==3.5.3
     # via
     #   django-q
     #   django-redis
+regex==2022.8.17
+    # via -r requirements.in
 requests==2.28.1
     # via
     #   coreapi