diff --git a/InvenTree/common/files.py b/InvenTree/common/files.py new file mode 100644 index 0000000000..1964e662e2 --- /dev/null +++ b/InvenTree/common/files.py @@ -0,0 +1,184 @@ +""" +Files management tools. +""" + +from rapidfuzz import fuzz +import tablib +import os + +from django.utils.translation import gettext_lazy as _ +from django.core.exceptions import ValidationError + +# from company.models import ManufacturerPart, SupplierPart + + +class FileManager: + """ Class for managing an uploaded file """ + + name = '' + + # Fields which are absolutely necessary for valid upload + REQUIRED_HEADERS = [ + 'Quantity' + ] + + # Fields which are used for part matching (only one of them is needed) + PART_MATCH_HEADERS = [ + 'Part_Name', + 'Part_IPN', + 'Part_ID', + ] + + # Fields which would be helpful but are not required + OPTIONAL_HEADERS = [ + ] + + EDITABLE_HEADERS = [ + ] + + HEADERS = REQUIRED_HEADERS + PART_MATCH_HEADERS + OPTIONAL_HEADERS + + def __init__(self, file, name=None): + """ Initialize the FileManager class with a user-uploaded file object """ + + # Set name + if name: + self.name = name + + # Process initial file + self.process(file) + + def process(self, file): + """ Process file """ + + self.data = None + + ext = os.path.splitext(file.name)[-1].lower() + + if ext in ['.csv', '.tsv', ]: + # These file formats need string decoding + raw_data = file.read().decode('utf-8') + elif ext in ['.xls', '.xlsx']: + raw_data = file.read() + else: + raise ValidationError(_(f'Unsupported file format: {ext}')) + + try: + self.data = tablib.Dataset().load(raw_data) + except tablib.UnsupportedFormat: + raise ValidationError(_(f'Error reading {self.name} file (invalid format)')) + except tablib.core.InvalidDimensions: + raise ValidationError(_(f'Error reading {self.name} file (incorrect dimension)')) + + def guess_header(self, header, threshold=80): + """ Try to match a header (from the file) to a list of known headers + + Args: + header - Header name to look for + threshold - Match threshold for fuzzy search + """ + + # Try for an exact match + for h in self.HEADERS: + if h == header: + return h + + # Try for a case-insensitive match + for h in self.HEADERS: + if h.lower() == header.lower(): + return h + + # Try for a case-insensitive match with space replacement + for h in self.HEADERS: + if h.lower() == header.lower().replace(' ', '_'): + return h + + # Finally, look for a close match using fuzzy matching + matches = [] + + for h in self.HEADERS: + ratio = fuzz.partial_ratio(header, h) + if ratio > threshold: + matches.append({'header': h, 'match': ratio}) + + if len(matches) > 0: + matches = sorted(matches, key=lambda item: item['match'], reverse=True) + return matches[0]['header'] + + return None + + def columns(self): + """ Return a list of headers for the thingy """ + headers = [] + + for header in self.data.headers: + headers.append({ + 'name': header, + 'guess': self.guess_header(header) + }) + + return headers + + def col_count(self): + if self.data is None: + return 0 + + return len(self.data.headers) + + def row_count(self): + """ Return the number of rows in the file. """ + + if self.data is None: + return 0 + + return len(self.data) + + def rows(self): + """ Return a list of all rows """ + rows = [] + + for i in range(self.row_count()): + + data = [item for item in self.get_row_data(i)] + + # Is the row completely empty? Skip! + empty = True + + for idx, item in enumerate(data): + if len(str(item).strip()) > 0: + empty = False + + try: + # Excel import casts number-looking-items into floats, which is annoying + if item == int(item) and not str(item) == str(int(item)): + data[idx] = int(item) + except ValueError: + pass + + # Skip empty rows + if empty: + continue + + row = { + 'data': data, + 'index': i + } + + rows.append(row) + + return rows + + def get_row_data(self, index): + """ Retrieve row data at a particular index """ + if self.data is None or index >= len(self.data): + return None + + return self.data[index] + + def get_row_dict(self, index): + """ Retrieve a dict object representing the data row at a particular offset """ + + if self.data is None or index >= len(self.data): + return None + + return self.data.dict[index] diff --git a/InvenTree/common/forms.py b/InvenTree/common/forms.py index f4ff082019..84e44f3a31 100644 --- a/InvenTree/common/forms.py +++ b/InvenTree/common/forms.py @@ -5,8 +5,6 @@ Django forms for interacting with common objects # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django import forms - from InvenTree.forms import HelperForm from .models import InvenTreeSetting diff --git a/InvenTree/common/views.py b/InvenTree/common/views.py index 056cbec32b..8220b00554 100644 --- a/InvenTree/common/views.py +++ b/InvenTree/common/views.py @@ -5,8 +5,12 @@ Django views for interacting with common models # -*- coding: utf-8 -*- from __future__ import unicode_literals +import os + from django.utils.translation import ugettext_lazy as _ from django.forms import CheckboxInput, Select +from django.conf import settings +from django.core.files.storage import FileSystemStorage from formtools.wizard.views import SessionWizardView @@ -106,7 +110,7 @@ class SettingEdit(AjaxUpdateView): class MultiStepFormView(SessionWizardView): - """ Setup basic methods of multi-step form + """ Setup basic methods of multi-step form form_list: list of forms form_steps_description: description for each form @@ -114,6 +118,17 @@ class MultiStepFormView(SessionWizardView): form_list = [] form_steps_description = [] + media_folder = '' + file_storage = FileSystemStorage(settings.MEDIA_ROOT) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if self.media_folder: + media_folder_abs = os.path.join(settings.MEDIA_ROOT, self.media_folder) + if not os.path.exists(media_folder_abs): + os.mkdir(media_folder_abs) + self.file_storage = FileSystemStorage(location=media_folder_abs) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -126,4 +141,3 @@ class MultiStepFormView(SessionWizardView): context.update({'description': description}) return context - \ No newline at end of file diff --git a/InvenTree/order/forms.py b/InvenTree/order/forms.py index b5b1aa2c96..5f6253217c 100644 --- a/InvenTree/order/forms.py +++ b/InvenTree/order/forms.py @@ -14,6 +14,8 @@ from InvenTree.forms import HelperForm from InvenTree.fields import RoundingDecimalFormField from InvenTree.fields import DatePickerFormField +from common.files import FileManager + import part.models from stock.models import StockLocation @@ -287,15 +289,29 @@ class EditSalesOrderAllocationForm(HelperForm): class UploadFile(forms.Form): - ''' Step 1 ''' - first_name = forms.CharField(max_length=100) + """ Step 1 """ + file = forms.FileField( + label=_('Order File'), + help_text=_('Select order file to upload'), + ) + + file_manager = None + + def clean_file(self): + file = self.cleaned_data['file'] + + # Create a FileManager object - will perform initial data validation + # (and raise a ValidationError if there is something wrong with the file) + self.file_manager = FileManager(file=file, name='order') + + return file class MatchField(forms.Form): - ''' Step 2 ''' + """ Step 2 """ last_name = forms.CharField(max_length=100) class MatchPart(forms.Form): - ''' Step 3 ''' - age = forms.IntegerField() \ No newline at end of file + """ Step 3 """ + age = forms.IntegerField() diff --git a/InvenTree/order/templates/order/po_upload.html b/InvenTree/order/templates/order/po_upload.html index 8563b70ff5..9650ee763d 100644 --- a/InvenTree/order/templates/order/po_upload.html +++ b/InvenTree/order/templates/order/po_upload.html @@ -17,7 +17,7 @@

{% trans "Step" %} {{ wizard.steps.step1 }} {% trans "of" %} {{ wizard.steps.count }} {% if description %}- {{ description }}{% endif %}

-
{% csrf_token %} +{% csrf_token %} {% load crispy_forms_tags %} {{ wizard.management_form }} diff --git a/InvenTree/order/views.py b/InvenTree/order/views.py index 92f5d54c33..ec9a4a4742 100644 --- a/InvenTree/order/views.py +++ b/InvenTree/order/views.py @@ -580,7 +580,7 @@ class PurchaseOrderUpload(MultiStepFormView): _("Select Parts"), ] template_name = "order/po_upload.html" - # file_storage = FileSystemStorage(location=os.path.join(settings.MEDIA_ROOT, 'file_uploads')) + media_folder = 'order_uploads/' def get_context_data(self, form, **kwargs): context = super().get_context_data(form=form, **kwargs) @@ -591,6 +591,24 @@ class PurchaseOrderUpload(MultiStepFormView): return context + def get_form_step_data(self, form): + # print(f'{self.steps.current=}\n{form.data=}') + return form.data + + def get_form_step_files(self, form): + # Check if user completed file upload + if self.steps.current == '0': + # Extract columns and rows from FileManager + self.extractDataFromFile(form.file_manager) + + return form.files + + def extractDataFromFile(self, file_manager): + """ Read data from the file """ + + self.columns = file_manager.columns() + self.rows = file_manager.rows() + def done(self, form_list, **kwargs): return HttpResponseRedirect(reverse('po-detail', kwargs={'pk': self.kwargs['pk']}))