Fixes and improvements for the part import wizard (#4127)

Made changes that resemble PR #4050 to the part import wizard to make the correct form show.
Added option to download a part import template file.
Increased the number of allowable formfield because the importer creates a lot of table fields when importing multiple parts at once.

InvenTree/InvenTree/settings.py

@@ -121,6 +121,9 @@ CORS_ORIGIN_WHITELIST = get_setting(
     default_value=[]
 )
 
+# Needed for the parts importer, directly impacts the maximum parts that can be uploaded
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000
+
 # Web URL endpoint for served static files
 STATIC_URL = '/static/'
 
@@ -376,9 +379,9 @@ for key in db_keys:
         db_config[key] = env_var
 
 # Check that required database configuration options are specified
-reqiured_keys = ['ENGINE', 'NAME']
+required_keys = ['ENGINE', 'NAME']
 
-for key in reqiured_keys:
+for key in required_keys:
     if key not in db_config:  # pragma: no cover
         error_msg = f'Missing required database configuration value {key}'
         logger.error(error_msg)

InvenTree/part/admin.py

@@ -104,6 +104,24 @@ class PartResource(InvenTreeResource):
         models.Part.objects.rebuild()
 
 
+class PartImportResource(InvenTreeResource):
+    """Class for managing Part data import/export."""
+
+    class Meta(PartResource.Meta):
+        """Metaclass definition"""
+        skip_unchanged = True
+        report_skipped = False
+        clean_model_instances = True
+        exclude = [
+            'id', 'category__name', 'creation_date', 'creation_user',
+            'pricing__overall_min', 'pricing__overall_max',
+            'bom_checksum', 'bom_checked_by', 'bom_checked_date',
+            'lft', 'rght', 'tree_id', 'level',
+            'metadata',
+            'barcode_data', 'barcode_hash',
+        ]
+
+
 class StocktakeInline(admin.TabularInline):
     """Inline for part stocktake data"""
     model = models.PartStocktake

InvenTree/part/part.py

@@ -0,0 +1,37 @@
+"""Functionality for Part import template.
+
+Primarily Part import tools.
+"""
+
+from InvenTree.helpers import DownloadFile, GetExportFormats
+
+from .admin import PartImportResource
+from .models import Part
+
+
+def IsValidPartFormat(fmt):
+    """Test if a file format specifier is in the valid list of part import template file formats."""
+    return fmt.strip().lower() in GetExportFormats()
+
+
+def MakePartTemplate(fmt):
+    """Generate a part import template file (for user download)."""
+    fmt = fmt.strip().lower()
+
+    if not IsValidPartFormat(fmt):
+        fmt = 'csv'
+
+    # Create an "empty" queryset, essentially.
+    # This will then export just the row headers!
+    query = Part.objects.filter(pk=None)
+
+    dataset = PartImportResource().export(
+        queryset=query,
+        importing=True
+    )
+
+    data = dataset.export(fmt)
+
+    filename = 'InvenTree_Part_Template.' + fmt
+
+    return DownloadFile(data, filename)

InvenTree/part/templates/part/import_wizard/ajax_part_upload.html

@@ -26,7 +26,7 @@
 
 {% else %}
     <div class='alert alert-danger alert-block' role='alert'>
-        {% trans "Unsuffitient privileges." %}
+        {% trans "Insufficient privileges." %}
     </div>
 {% endif %}
 </div>

InvenTree/part/templates/part/import_wizard/match_fields.html

@@ -1,2 +1,99 @@
 {% extends "part/import_wizard/part_upload.html" %}
-{% include "patterns/wizard/match_fields.html" %}
+{% load inventree_extras %}
+{% load i18n %}
+{% load static %}
+
+{% block form_alert %}
+{% if missing_columns and missing_columns|length > 0 %}
+<div class='alert alert-danger alert-block' style='margin-top:12px;' role='alert'>
+    {% trans "Missing selections for the following required columns" %}:
+    <br>
+    <ul>
+        {% for col in missing_columns %}
+        <li>{{ col }}</li>
+        {% endfor %}
+    </ul>
+</div>
+{% endif %}
+{% if duplicates and duplicates|length > 0 %}
+<div class='alert alert-danger alert-block' role='alert'>
+    {% trans "Duplicate selections found, see below. Fix them then retry submitting." %}
+</div>
+{% endif %}
+{% endblock form_alert %}
+
+{% block form_buttons_top %}
+    {% if wizard.steps.prev %}
+    <button name='wizard_goto_step' type='submit' value='{{ wizard.steps.prev }}' class='save btn btn-outline-secondary'>{% trans "Previous Step" %}</button>
+    {% endif %}
+    <button type='submit' class='save btn btn-outline-secondary'>{% trans "Submit Selections" %}</button>
+{% endblock form_buttons_top %}
+
+{% block form_content %}
+    <thead>
+        <tr>
+            <th>{% trans "File Fields" %}</th>
+            <th></th>
+            {% for col in form %}
+            <th>
+                <div>
+                    <input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
+                    {{ col.name }}
+                    <button class='btn btn-outline-secondary btn-remove' onClick='removeColFromBomWizard()' id='del_col_{{ forloop.counter0 }}' style='display: inline; float: right;' title='{% trans "Remove column" %}'>
+                        <span col_id='{{ forloop.counter0 }}' class='fas fa-trash-alt icon-red'></span>
+                    </button>
+                </div>
+            </th>
+            {% endfor %}
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>{% trans "Match Fields" %}</td>
+            <td></td>
+            {% for col in form %}
+            <td>
+                {{ col }}
+                {% for duplicate in duplicates %}
+                    {% if duplicate == col.value %}
+                    <div class='alert alert-danger alert-block text-center' role='alert' style='padding:2px; margin-top:6px; margin-bottom:2px'>
+                        <strong>{% trans "Duplicate selection" %}</strong>
+                    </div>
+                    {% endif %}
+                {% endfor %}
+            </td>
+            {% endfor %}
+        </tr>
+        {% for row in rows %}
+        {% with forloop.counter as row_index %}
+        <tr>
+            <td style='width: 32px;'>
+                <button class='btn btn-outline-secondary btn-remove' onClick='removeRowFromBomWizard()' id='del_row_{{ row_index }}' style='display: inline; float: left;' title='{% trans "Remove row" %}'>
+                    <span row_id='{{ row_index }}' class='fas fa-trash-alt icon-red'></span>
+                </button>
+            </td>
+            <td style='text-align: left;'>{{ row_index }}</td>
+            {% for item in row.data %}
+            <td>
+                <input type='hidden' name='row_{{ row_index }}_col_{{ forloop.counter0 }}' value='{{ item }}'/>
+                {{ item }}
+            </td>
+            {% endfor %}
+        </tr>
+        {% endwith %}
+        {% endfor %}
+    </tbody>
+{% endblock form_content %}
+
+{% block form_buttons_bottom %}
+{% endblock form_buttons_bottom %}
+
+{% block js_ready %}
+{{ block.super }}
+
+$('.fieldselect').select2({
+    width: '100%',
+    matcher: partialMatcher,
+});
+
+{% endblock %}

InvenTree/part/templates/part/import_wizard/part_upload.html

@@ -11,8 +11,61 @@
 
 {% block content %}
     {% trans "Import Parts from File" as header_text %}
-    {% trans "Unsuffitient privileges." as error_text %}
-    {% include "patterns/wizard/upload.html" with header_text=header_text upload_go_ahead=roles.part.change error_text=error_text %}
+    {% trans "Insufficient privileges." as error_text %}
+
+    <div class='panel' id='panel-upload-file'>
+        <div class='panel-heading'>
+            <h4>
+                {{ header_text }}
+                {{ wizard.form.media }}
+            </h4>
+        </div>
+        <div class='panel-content'>
+        {% if roles.part.change %}
+
+            <p>{% blocktrans with step=wizard.steps.step1 count=wizard.steps.count %}Step {{step}} of {{count}}{% endblocktrans %}
+            {% if description %}- {{ description }}{% endif %}</p>
+
+            {% block form_alert %}
+            <div class='alert alert-info alert-block'>
+                <strong>{% trans "Requirements for part import" %}:</strong>
+                <ul>
+                    <li>{% trans "The part import file must contain the required named columns as provided in the " %} <strong><a href='#' id='part-template-download'>{% trans "Part Import Template" %}</a></strong></li>
+                </ul>
+            </div>
+            {% endblock form_alert %}
+
+            <form action='' method='post' class='js-modal-form' enctype='multipart/form-data'>
+            {% csrf_token %}
+            {% load crispy_forms_tags %}
+
+            {% block form_buttons_top %}
+            {% endblock form_buttons_top %}
+
+            <div style='overflow-x:scroll;'>
+                <table class='table table-striped' style='margin-top: 12px; margin-bottom: 0px; table-layout: auto; width: 100%;'>
+                {{ wizard.management_form }}
+                {% block form_content %}
+                {% crispy wizard.form %}
+                {% endblock form_content %}
+                </table>
+            </div>
+
+            {% block form_buttons_bottom %}
+            {% if wizard.steps.prev %}
+            <button name='wizard_goto_step' type='submit' value='{{ wizard.steps.prev }}' class='save btn btn-outline-secondary'>{% trans "Previous Step" %}</button>
+            {% endif %}
+            <button type='submit' class='save btn btn-outline-secondary'>{% trans "Upload File" %}</button>
+            </form>
+            {% endblock form_buttons_bottom %}
+
+        {% else %}
+            <div class='alert alert-danger alert-block' role='alert'>
+                {{ error_text }}
+            </div>
+        {% endif %}
+        </div>
+    </div>
 {% endblock %}
 
 {% block js_ready %}
@@ -20,4 +73,44 @@
 
 enableSidebar('partupload');
 
+$('#part-template-download').click(function() {
+    downloadPartImportTemplate();
+});
+
+function downloadPartImportTemplate(options={}) {
+
+    var format = options.format;
+
+    if (!format) {
+        format = inventreeLoad('part-import-format', 'csv');
+    }
+
+    constructFormBody({}, {
+        title: '{% trans "Download Part Import Template" %}',
+        fields: {
+            format: {
+                label: '{% trans "Format" %}',
+                help_text: '{% trans "Select file format" %}',
+                required: true,
+                type: 'choice',
+                value: format,
+                choices: exportFormatOptions(),
+            }
+        },
+        onSubmit: function(fields, opts) {
+            var format = getFormFieldValue('format', fields['format'], opts);
+
+            // Save the format for next time
+            inventreeSave('part-import-format', format);
+
+            // Hide the modal
+            $(opts.modal).modal('hide');
+
+            // Download the file
+            location.href = `{% url "part-template-download" %}?format=${format}`;
+
+        }
+    });
+}
+
 {% endblock %}

InvenTree/part/urls.py

@@ -36,7 +36,8 @@ category_urls = [
 part_urls = [
 
     # Upload a part
-    re_path(r'^import/', views.PartImport.as_view(), name='part-import'),
+    re_path(r'^import/$', views.PartImport.as_view(), name='part-import'),
+    re_path(r'^import/?', views.PartImportTemplate.as_view(), name='part-template-download'),
     re_path(r'^import-api/', views.PartImportAjax.as_view(), name='api-part-import'),
 
     # Download a BOM upload template

InvenTree/part/views.py

@@ -15,7 +15,7 @@ from common.files import FileManager
 from common.models import InvenTreeSetting
 from common.views import FileManagementAjaxView, FileManagementFormView
 from company.models import SupplierPart
-from InvenTree.helpers import str2bool
+from InvenTree.helpers import str2bool, str2int
 from InvenTree.views import (AjaxUpdateView, AjaxView, InvenTreeRoleMixin,
                              QRCodeView)
 from plugin.views import InvenTreePluginViewMixin
@@ -25,6 +25,7 @@ from . import forms as part_forms
 from . import settings as part_settings
 from .bom import ExportBom, IsValidBOMFormat, MakeBomTemplate
 from .models import Part, PartCategory
+from .part import MakePartTemplate
 
 
 class PartIndex(InvenTreeRoleMixin, ListView):
@@ -90,11 +91,12 @@ class PartImport(FileManagementFormView):
             'Assembly',
             'Component',
             'is_template',
-            'Purchaseable',
+            'Purchasable',
             'Salable',
             'Trackable',
             'Virtual',
             'Stock',
+            'Image',
         ]
 
     name = 'part'
@@ -135,6 +137,7 @@ class PartImport(FileManagementFormView):
         'trackable': 'trackable',
         'virtual': 'virtual',
         'stock': 'stock',
+        'image': 'image',
     }
     file_manager_class = PartFileManager
 
@@ -144,14 +147,14 @@ class PartImport(FileManagementFormView):
         self.allowed_items = {}
         self.matches = {}
 
-        self.allowed_items['Category'] = PartCategory.objects.all()
-        self.matches['Category'] = ['name__contains']
-        self.allowed_items['default_location'] = StockLocation.objects.all()
-        self.matches['default_location'] = ['name__contains']
+        self.allowed_items['Category'] = PartCategory.objects.all().exclude(structural=True)
+        self.matches['Category'] = ['name__icontains']
+        self.allowed_items['default_location'] = StockLocation.objects.all().exclude(structural=True)
+        self.matches['default_location'] = ['name__icontains']
         self.allowed_items['default_supplier'] = SupplierPart.objects.all()
-        self.matches['default_supplier'] = ['SKU__contains']
-        self.allowed_items['variant_of'] = Part.objects.all()
-        self.matches['variant_of'] = ['name__contains']
+        self.matches['default_supplier'] = ['SKU__icontains']
+        self.allowed_items['variant_of'] = Part.objects.all().exclude(is_template=False)
+        self.matches['variant_of'] = ['name__icontains']
 
         # setup
         self.file_manager.setup()
@@ -210,8 +213,8 @@ class PartImport(FileManagementFormView):
                 IPN=part_data.get('ipn', None),
                 revision=part_data.get('revision', None),
                 link=part_data.get('link', None),
-                default_expiry=part_data.get('default_expiry', 0),
-                minimum_stock=part_data.get('minimum_stock', 0),
+                default_expiry=str2int(part_data.get('default_expiry'), 0),
+                minimum_stock=str2int(part_data.get('minimum_stock'), 0),
                 units=part_data.get('units', None),
                 notes=part_data.get('notes', None),
                 category=optional_matches['Category'],
@@ -219,8 +222,8 @@ class PartImport(FileManagementFormView):
                 default_supplier=optional_matches['default_supplier'],
                 variant_of=optional_matches['variant_of'],
                 active=str2bool(part_data.get('active', True)),
-                base_cost=part_data.get('base_cost', 0),
-                multiple=part_data.get('multiple', 1),
+                base_cost=str2int(part_data.get('base_cost'), 0),
+                multiple=str2int(part_data.get('multiple'), 1),
                 assembly=str2bool(part_data.get('assembly', part_settings.part_assembly_default())),
                 component=str2bool(part_data.get('component', part_settings.part_component_default())),
                 is_template=str2bool(part_data.get('is_template', part_settings.part_template_default())),
@@ -228,7 +231,14 @@ class PartImport(FileManagementFormView):
                 salable=str2bool(part_data.get('salable', part_settings.part_salable_default())),
                 trackable=str2bool(part_data.get('trackable', part_settings.part_trackable_default())),
                 virtual=str2bool(part_data.get('virtual', part_settings.part_virtual_default())),
+                image=part_data.get('image', None),
             )
+
+            # check if theres a category assigned, if not skip this part or else bad things happen
+            if not optional_matches['Category']:
+                import_error.append(_("Can't import part {name} because there is no category assigned").format(name=new_part.name))
+                continue
+
             try:
                 new_part.save()
 
@@ -240,6 +250,7 @@ class PartImport(FileManagementFormView):
                         quantity=int(part_data.get('stock', 1)),
                     )
                     stock.save()
+
                 import_done += 1
             except ValidationError as _e:
                 import_error.append(', '.join(set(_e.messages)))
@@ -249,12 +260,25 @@ class PartImport(FileManagementFormView):
             alert = f"<strong>{_('Part-Import')}</strong><br>{_('Imported {n} parts').format(n=import_done)}"
             messages.success(self.request, alert)
         if import_error:
-            error_text = '\n'.join([f'<li><strong>x{import_error.count(a)}</strong>: {a}</li>' for a in set(import_error)])
+            error_text = '\n'.join([f'<li><strong>{import_error.count(a)}</strong>: {a}</li>' for a in set(import_error)])
             messages.error(self.request, f"<strong>{_('Some errors occured:')}</strong><br><ul>{error_text}</ul>")
 
         return HttpResponseRedirect(reverse('part-index'))
 
 
+class PartImportTemplate(AjaxView):
+    """Provide a part import template file for download.
+
+    - Generates a template file in the provided format e.g. ?format=csv
+    """
+
+    def get(self, request, *args, **kwargs):
+        """Perform a GET request to download the 'Part import' template"""
+        export_format = request.GET.get('format', 'csv')
+
+        return MakePartTemplate(export_format)
+
+
 class PartImportAjax(FileManagementAjaxView, PartImport):
     """Multi-step form wizard for importing Part data"""
     ajax_form_steps_template = [