Merge pull request #1760 from eeintech/bom_import

Converted BOM import to new multi-step form framework
This commit is contained in:
Oliver 2021-07-10 13:47:25 +10:00 committed by GitHub
commit f6d5bd4ed8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 423 additions and 931 deletions

View File

@ -3,14 +3,9 @@ Functionality for Bill of Material (BOM) management.
Primarily BOM upload tools. Primarily BOM upload tools.
""" """
from rapidfuzz import fuzz
import tablib
import os
from collections import OrderedDict from collections import OrderedDict
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.core.exceptions import ValidationError
from InvenTree.helpers import DownloadFile, GetExportFormats from InvenTree.helpers import DownloadFile, GetExportFormats
@ -326,174 +321,3 @@ def ExportBom(part, fmt='csv', cascade=False, max_levels=None, parameter_data=Fa
filename = '{n}_BOM.{fmt}'.format(n=part.full_name, fmt=fmt) filename = '{n}_BOM.{fmt}'.format(n=part.full_name, fmt=fmt)
return DownloadFile(data, filename) return DownloadFile(data, filename)
class BomUploadManager:
""" Class for managing an uploaded BOM file """
# Fields which are absolutely necessary for valid upload
REQUIRED_HEADERS = [
'Quantity'
]
# Fields which are used for part matching (only one of them is needed)
PART_MATCH_HEADERS = [
'Part_Name',
'Part_IPN',
'Part_ID',
]
# Fields which would be helpful but are not required
OPTIONAL_HEADERS = [
'Reference',
'Note',
'Overage',
]
EDITABLE_HEADERS = [
'Reference',
'Note',
'Overage'
]
HEADERS = REQUIRED_HEADERS + PART_MATCH_HEADERS + OPTIONAL_HEADERS
def __init__(self, bom_file):
""" Initialize the BomUpload class with a user-uploaded file object """
self.process(bom_file)
def process(self, bom_file):
""" Process a BOM file """
self.data = None
ext = os.path.splitext(bom_file.name)[-1].lower()
if ext in ['.csv', '.tsv', ]:
# These file formats need string decoding
raw_data = bom_file.read().decode('utf-8')
elif ext in ['.xls', '.xlsx']:
raw_data = bom_file.read()
else:
raise ValidationError({'bom_file': _('Unsupported file format: {f}').format(f=ext)})
try:
self.data = tablib.Dataset().load(raw_data)
except tablib.UnsupportedFormat:
raise ValidationError({'bom_file': _('Error reading BOM file (invalid data)')})
except tablib.core.InvalidDimensions:
raise ValidationError({'bom_file': _('Error reading BOM file (incorrect row size)')})
def guess_header(self, header, threshold=80):
""" Try to match a header (from the file) to a list of known headers
Args:
header - Header name to look for
threshold - Match threshold for fuzzy search
"""
# Try for an exact match
for h in self.HEADERS:
if h == header:
return h
# Try for a case-insensitive match
for h in self.HEADERS:
if h.lower() == header.lower():
return h
# Try for a case-insensitive match with space replacement
for h in self.HEADERS:
if h.lower() == header.lower().replace(' ', '_'):
return h
# Finally, look for a close match using fuzzy matching
matches = []
for h in self.HEADERS:
ratio = fuzz.partial_ratio(header, h)
if ratio > threshold:
matches.append({'header': h, 'match': ratio})
if len(matches) > 0:
matches = sorted(matches, key=lambda item: item['match'], reverse=True)
return matches[0]['header']
return None
def columns(self):
""" Return a list of headers for the thingy """
headers = []
for header in self.data.headers:
headers.append({
'name': header,
'guess': self.guess_header(header)
})
return headers
def col_count(self):
if self.data is None:
return 0
return len(self.data.headers)
def row_count(self):
""" Return the number of rows in the file. """
if self.data is None:
return 0
return len(self.data)
def rows(self):
""" Return a list of all rows """
rows = []
for i in range(self.row_count()):
data = [item for item in self.get_row_data(i)]
# Is the row completely empty? Skip!
empty = True
for idx, item in enumerate(data):
if len(str(item).strip()) > 0:
empty = False
try:
# Excel import casts number-looking-items into floats, which is annoying
if item == int(item) and not str(item) == str(int(item)):
data[idx] = int(item)
except ValueError:
pass
# Skip empty rows
if empty:
continue
row = {
'data': data,
'index': i
}
rows.append(row)
return rows
def get_row_data(self, index):
""" Retrieve row data at a particular index """
if self.data is None or index >= len(self.data):
return None
return self.data[index]
def get_row_dict(self, index):
""" Retrieve a dict object representing the data row at a particular offset """
if self.data is None or index >= len(self.data):
return None
return self.data.dict[index]

View File

@ -11,10 +11,11 @@ from django.utils.translation import ugettext_lazy as _
from mptt.fields import TreeNodeChoiceField from mptt.fields import TreeNodeChoiceField
from InvenTree.forms import HelperForm from InvenTree.forms import HelperForm
from InvenTree.helpers import GetExportFormats from InvenTree.helpers import GetExportFormats, clean_decimal
from InvenTree.fields import RoundingDecimalFormField from InvenTree.fields import RoundingDecimalFormField
import common.models import common.models
from common.forms import MatchItemForm
from .models import Part, PartCategory, PartRelated from .models import Part, PartCategory, PartRelated
from .models import BomItem from .models import BomItem
@ -143,16 +144,28 @@ class BomValidateForm(HelperForm):
] ]
class BomUploadSelectFile(HelperForm): class BomMatchItemForm(MatchItemForm):
""" Form for importing a BOM. Provides a file input box for upload """ """ Override MatchItemForm fields """
bom_file = forms.FileField(label=_('BOM file'), required=True, help_text=_("Select BOM file to upload")) def get_special_field(self, col_guess, row, file_manager):
""" Set special fields """
class Meta: # set quantity field
model = Part if 'quantity' in col_guess.lower():
fields = [ return forms.CharField(
'bom_file', required=False,
] widget=forms.NumberInput(attrs={
'name': 'quantity' + str(row['index']),
'class': 'numberinput',
'type': 'number',
'min': '0',
'step': 'any',
'value': clean_decimal(row.get('quantity', '')),
})
)
# return default
return super().get_special_field(col_guess, row, file_manager)
class CreatePartRelatedForm(HelperForm): class CreatePartRelatedForm(HelperForm):

View File

@ -11,6 +11,12 @@
{% endblock %} {% endblock %}
{% block details %} {% block details %}
{% if roles.part.change != True and editing_enabled %}
<div class='alert alert-danger alert-block'>
{% trans "You do not have permission to edit the BOM." %}
</div>
{% else %}
{% if part.bom_checked_date %} {% if part.bom_checked_date %}
{% if part.is_bom_valid %} {% if part.is_bom_valid %}
<div class='alert alert-block alert-info'> <div class='alert alert-block alert-info'>
@ -72,6 +78,7 @@
<table class='table table-bom table-condensed' data-toolbar="#button-toolbar" id='bom-table'> <table class='table table-bom table-condensed' data-toolbar="#button-toolbar" id='bom-table'>
</table> </table>
{% endif %}
{% endblock %} {% endblock %}

View File

@ -0,0 +1,99 @@
{% extends "part/bom_upload/upload_file.html" %}
{% load inventree_extras %}
{% load i18n %}
{% load static %}
{% block form_alert %}
{% if missing_columns and missing_columns|length > 0 %}
<div class='alert alert-danger alert-block' role='alert'>
{% trans "Missing selections for the following required columns" %}:
<br>
<ul>
{% for col in missing_columns %}
<li>{{ col }}</li>
{% endfor %}
</ul>
</div>
{% endif %}
{% if duplicates and duplicates|length > 0 %}
<div class='alert alert-danger alert-block' role='alert'>
{% trans "Duplicate selections found, see below. Fix them then retry submitting." %}
</div>
{% endif %}
{% endblock form_alert %}
{% block form_buttons_top %}
{% if wizard.steps.prev %}
<button name="wizard_goto_step" type="submit" value="{{ wizard.steps.prev }}" class="save btn btn-default">{% trans "Previous Step" %}</button>
{% endif %}
<button type="submit" class="save btn btn-default">{% trans "Submit Selections" %}</button>
{% endblock form_buttons_top %}
{% block form_content %}
<thead>
<tr>
<th>{% trans "File Fields" %}</th>
<th></th>
{% for col in form %}
<th>
<div>
<input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
{{ col.name }}
<button class='btn btn-default btn-remove' onClick='removeColFromBomWizard()' id='del_col_{{ forloop.counter0 }}' style='display: inline; float: right;' title='{% trans "Remove column" %}'>
<span col_id='{{ forloop.counter0 }}' class='fas fa-trash-alt icon-red'></span>
</button>
</div>
</th>
{% endfor %}
</tr>
</thead>
<tbody>
<tr>
<td>{% trans "Match Fields" %}</td>
<td></td>
{% for col in form %}
<td>
{{ col }}
{% for duplicate in duplicates %}
{% if duplicate == col.value %}
<div class='alert alert-danger alert-block text-center' role='alert' style='padding:2px; margin-top:6px; margin-bottom:2px'>
<b>{% trans "Duplicate selection" %}</b>
</div>
{% endif %}
{% endfor %}
</td>
{% endfor %}
</tr>
{% for row in rows %}
{% with forloop.counter as row_index %}
<tr>
<td style='width: 32px;'>
<button class='btn btn-default btn-remove' onClick='removeRowFromBomWizard()' id='del_row_{{ row_index }}' style='display: inline; float: left;' title='{% trans "Remove row" %}'>
<span row_id='{{ row_index }}' class='fas fa-trash-alt icon-red'></span>
</button>
</td>
<td style='text-align: left;'>{{ row_index }}</td>
{% for item in row.data %}
<td>
<input type='hidden' name='row_{{ row_index }}_col_{{ forloop.counter0 }}' value='{{ item }}'/>
{{ item }}
</td>
{% endfor %}
</tr>
{% endwith %}
{% endfor %}
</tbody>
{% endblock form_content %}
{% block form_buttons_bottom %}
{% endblock form_buttons_bottom %}
{% block js_ready %}
{{ block.super }}
$('.fieldselect').select2({
width: '100%',
matcher: partialMatcher,
});
{% endblock %}

View File

@ -0,0 +1,127 @@
{% extends "part/bom_upload/upload_file.html" %}
{% load inventree_extras %}
{% load i18n %}
{% load static %}
{% load crispy_forms_tags %}
{% block form_alert %}
{% if form.errors %}
{% endif %}
{% if form_errors %}
<div class='alert alert-danger alert-block' role='alert'>
{% trans "Errors exist in the submitted data" %}
</div>
{% endif %}
{% endblock form_alert %}
{% block form_buttons_top %}
{% if wizard.steps.prev %}
<button name="wizard_goto_step" type="submit" value="{{ wizard.steps.prev }}" class="save btn btn-default">{% trans "Previous Step" %}</button>
{% endif %}
<button type="submit" class="save btn btn-default">{% trans "Submit Selections" %}</button>
{% endblock form_buttons_top %}
{% block form_content %}
<thead>
<tr>
<th></th>
<th>{% trans "Row" %}</th>
<th>{% trans "Select Part" %}</th>
<th>{% trans "Reference" %}</th>
<th>{% trans "Quantity" %}</th>
{% for col in columns %}
{% if col.guess != 'Quantity' %}
<th>
<input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
<input type='hidden' name='col_guess_{{ forloop.counter0 }}' value='{{ col.guess }}'/>
{% if col.guess %}
{{ col.guess }}
{% else %}
{{ col.name }}
{% endif %}
</th>
{% endif %}
{% endfor %}
</tr>
</thead>
<tbody>
<tr></tr> {% comment %} Dummy row for javascript del_row method {% endcomment %}
{% for row in rows %}
<tr {% if row.errors %} style='background: #ffeaea;'{% endif %} part-select='#select_part_{{ row.index }}'>
<td>
<button class='btn btn-default btn-remove' onClick='removeRowFromBomWizard()' id='del_row_{{ row.index }}' style='display: inline; float: right;' title='{% trans "Remove row" %}'>
<span row_id='{{ row.index }}' class='fas fa-trash-alt icon-red'></span>
</button>
</td>
<td>
{% add row.index 1 %}
</td>
<td>
{% for field in form.visible_fields %}
{% if field.name == row.item_select %}
{{ field }}
{% endif %}
{% endfor %}
{% if row.errors.part %}
<p class='help-inline'>{{ row.errors.part }}</p>
{% endif %}
</td>
<td>
{% for field in form.visible_fields %}
{% if field.name == row.reference %}
{{ field|as_crispy_field }}
{% endif %}
{% endfor %}
{% if row.errors.reference %}
<p class='help-inline'>{{ row.errors.reference }}</p>
{% endif %}
</td>
<td>
{% for field in form.visible_fields %}
{% if field.name == row.quantity %}
{{ field|as_crispy_field }}
{% endif %}
{% endfor %}
{% if row.errors.quantity %}
<p class='help-inline'>{{ row.errors.quantity }}</p>
{% endif %}
</td>
{% for item in row.data %}
{% if item.column.guess != 'Quantity' %}
<td>
{% if item.column.guess == 'Overage' %}
{% for field in form.visible_fields %}
{% if field.name == row.overage %}
{{ field|as_crispy_field }}
{% endif %}
{% endfor %}
{% elif item.column.guess == 'Note' %}
{% for field in form.visible_fields %}
{% if field.name == row.note %}
{{ field|as_crispy_field }}
{% endif %}
{% endfor %}
{% else %}
{{ item.cell }}
{% endif %}
<input type='hidden' name='row_{{ row.index }}_col_{{ forloop.counter0 }}' value='{{ item.cell }}'/>
</td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</tbody>
{% endblock form_content %}
{% block form_buttons_bottom %}
{% endblock form_buttons_bottom %}
{% block js_ready %}
{{ block.super }}
$('.bomselect').select2({
dropdownAutoWidth: true,
matcher: partialMatcher,
});
{% endblock %}

View File

@ -1,94 +0,0 @@
{% extends "part/part_base.html" %}
{% load static %}
{% load i18n %}
{% load inventree_extras %}
{% block menubar %}
{% include "part/navbar.html" with tab='bom' %}
{% endblock %}
{% block heading %}
{% trans "Upload Bill of Materials" %}
{% endblock %}
{% block details %}
<p>{% trans "Step 2 - Select Fields" %}</p>
<hr>
{% if missing_columns and missing_columns|length > 0 %}
<div class='alert alert-danger alert-block' role='alert'>
{% trans "Missing selections for the following required columns" %}:
<br>
<ul>
{% for col in missing_columns %}
<li>{{ col }}</li>
{% endfor %}
</ul>
</div>
{% endif %}
<form method="post" action='' class='js-modal-form' enctype="multipart/form-data">
<button type="submit" class="save btn btn-default">{% trans "Submit Selections" %}</button>
{% csrf_token %}
<input type='hidden' name='form_step' value='select_fields'/>
<table class='table table-striped'>
<thead>
<tr>
<th></th>
<th>{% trans "File Fields" %}</th>
{% for col in bom_columns %}
<th>
<div>
<input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
{{ col.name }}
<button class='btn btn-default btn-remove' onClick='removeColFromBomWizard()' id='del_col_{{ forloop.counter0 }}' style='display: inline; float: right;' title='{% trans "Remove column" %}'>
<span col_id='{{ forloop.counter0 }}' class='fas fa-trash-alt icon-red'></span>
</button>
</div>
</th>
{% endfor %}
</tr>
</thead>
<tbody>
<tr>
<td></td>
<td>{% trans "Match Fields" %}</td>
{% for col in bom_columns %}
<td>
<select class='select' id='id_col_{{ forloop.counter0 }}' name='col_guess_{{ forloop.counter0 }}'>
<option value=''>---------</option>
{% for req in bom_headers %}
<option value='{{ req }}'{% if req == col.guess %}selected='selected'{% endif %}>{{ req }}</option>
{% endfor %}
</select>
{% if col.duplicate %}
<p class='help-inline'>{% trans "Duplicate column selection" %}</p>
{% endif %}
</td>
{% endfor %}
</tr>
{% for row in bom_rows %}
<tr>
<td>
<button class='btn btn-default btn-remove' onClick='removeRowFromBomWizard()' id='del_row_{{ forloop.counter }}' style='display: inline; float: right;' title='{% trans "Remove row" %}'>
<span row_id='{{ forloop.counter }}' class='fas fa-trash-alt icon-red'></span>
</button>
</td>
<td>{{ forloop.counter }}</td>
{% for item in row.data %}
<td>
<input type='hidden' name='row_{{ row.index }}_col_{{ forloop.counter0 }}' value='{{ item.cell }}'/>
{{ item.cell }}
</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
</form>
{% endblock %}

View File

@ -1,121 +0,0 @@
{% extends "part/part_base.html" %}
{% load static %}
{% load i18n %}
{% load inventree_extras %}
{% block menubar %}
{% include "part/navbar.html" with tab="bom" %}
{% endblock %}
{% block heading %}
{% trans "Upload Bill of Materials" %}
{% endblock %}
{% block details %}
<p>{% trans "Step 3 - Select Parts" %}</p>
<hr>
{% if form_errors %}
<div class='alert alert-danger alert-block' role='alert'>
{% trans "Errors exist in the submitted data" %}
</div>
{% endif %}
<form method="post" action='' class='js-modal-form' enctype="multipart/form-data">
<button type="submit" class="save btn btn-default">{% trans "Submit BOM" %}</button>
{% csrf_token %}
{% load crispy_forms_tags %}
<input type='hidden' name='form_step' value='select_parts'/>
<table class='table table-striped'>
<thead>
<tr>
<th></th>
<th></th>
<th>{% trans "Row" %}</th>
<th>{% trans "Select Part" %}</th>
{% for col in bom_columns %}
<th>
<input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
<input type='hidden' name='col_guess_{{ forloop.counter0 }}' value='{{ col.guess }}'/>
{% if col.guess %}
{{ col.guess }}
{% else %}
{{ col.name }}
{% endif %}
</th>
{% endfor %}
</tr>
</thead>
<tbody>
{% for row in bom_rows %}
<tr {% if row.errors %} style='background: #ffeaea;'{% endif %} part-name='{{ row.part_name }}' part-description='{{ row.description }}' part-select='#select_part_{{ row.index }}'>
<td>
<button class='btn btn-default btn-remove' onClick='removeRowFromBomWizard()' id='del_row_{{ forloop.counter }}' style='display: inline; float: right;' title='{% trans "Remove row" %}'>
<span row_id='{{ forloop.counter }}' class='fas fa-trash-alt icon-red'></span>
</button>
</td>
<td></td>
<td>{% add row.index 1 %}</td>
<td>
<button class='btn btn-default btn-create' onClick='newPartFromBomWizard()' id='new_part_row_{{ row.index }}' title='{% trans "Create new part" %}' type='button'>
<span row_id='{{ row.index }}' class='fas fa-plus icon-green'/>
</button>
<select class='select bomselect' id='select_part_{{ row.index }}' name='part_{{ row.index }}'>
<option value=''>--- {% trans "Select Part" %} ---</option>
{% for part in row.part_options %}
<option value='{{ part.id }}' {% if part.id == row.part.id %} selected='selected' {% elif part.id == row.part_match.id %} selected='selected' {% endif %}>
{{ part }}
</option>
{% endfor %}
</select>
{% if row.errors.part %}
<p class='help-inline'>{{ row.errors.part }}</p>
{% endif %}
</td>
{% for item in row.data %}
<td>
{% if item.column.guess == 'Part' %}
<i>{{ item.cell }}</i>
{% if row.errors.part %}
<p class='help-inline'>{{ row.errors.part }}</p>
{% endif %}
{% elif item.column.guess == 'Quantity' %}
<input name='quantity_{{ row.index }}' class='numberinput' type='number' min='1' step='any' value='{% decimal row.quantity %}'/>
{% if row.errors.quantity %}
<p class='help-inline'>{{ row.errors.quantity }}</p>
{% endif %}
{% elif item.column.guess == 'Reference' %}
<input name='reference_{{ row.index }}' value='{{ row.reference }}'/>
{% elif item.column.guess == 'Note' %}
<input name='notes_{{ row.index }}' value='{{ row.notes }}'/>
{% elif item.column.guess == 'Overage' %}
<input name='overage_{{ row.index }}' value='{{ row.overage }}'/>
{% else %}
{{ item.cell }}
{% endif %}
<input type='hidden' name='row_{{ row.index }}_col_{{ forloop.counter0 }}' value='{{ item.cell }}'/>
</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
</form>
{% endblock %}
{% block js_ready %}
{{ block.super }}
$('.bomselect').select2({
dropdownAutoWidth: true,
matcher: partialMatcher,
});
{% endblock %}

View File

@ -8,13 +8,12 @@
{% endblock %} {% endblock %}
{% block heading %} {% block heading %}
{% trans "Upload Bill of Materials" %} {% trans "Upload BOM File" %}
{% endblock %} {% endblock %}
{% block details %} {% block details %}
<p>{% trans "Step 1 - Select BOM File" %}</p> {% block form_alert %}
<div class='alert alert-info alert-block'> <div class='alert alert-info alert-block'>
<b>{% trans "Requirements for BOM upload" %}:</b> <b>{% trans "Requirements for BOM upload" %}:</b>
<ul> <ul>
@ -22,16 +21,31 @@
<li>{% trans "Each part must already exist in the database" %}</li> <li>{% trans "Each part must already exist in the database" %}</li>
</ul> </ul>
</div> </div>
{% endblock %}
<form method="post" action='' class='js-modal-form' enctype="multipart/form-data"> <p>{% blocktrans with step=wizard.steps.step1 count=wizard.steps.count %}Step {{step}} of {{count}}{% endblocktrans %}
<button type="submit" class="save btn btn-default">{% trans 'Upload File' %}</button> {% if description %}- {{ description }}{% endif %}</p>
{% csrf_token %}
{% load crispy_forms_tags %}
<input type='hidden' name='form_step' value='select_file'/> <form action="" method="post" class='js-modal-form' enctype="multipart/form-data">
{% csrf_token %}
{% load crispy_forms_tags %}
{% crispy form %} {% block form_buttons_top %}
{% endblock form_buttons_top %}
<table class='table table-striped' style='margin-top: 12px; margin-bottom: 0px'>
{{ wizard.management_form }}
{% block form_content %}
{% crispy wizard.form %}
{% endblock form_content %}
</table>
{% block form_buttons_bottom %}
{% if wizard.steps.prev %}
<button name="wizard_goto_step" type="submit" value="{{ wizard.steps.prev }}" class="save btn btn-default">{% trans "Previous Step" %}</button>
{% endif %}
<button type="submit" class="save btn btn-default">{% trans "Upload File" %}</button>
</form> </form>
{% endblock form_buttons_bottom %}
{% endblock %} {% endblock %}

View File

@ -13,7 +13,7 @@ from django.shortcuts import get_object_or_404
from django.shortcuts import HttpResponseRedirect from django.shortcuts import HttpResponseRedirect
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.urls import reverse, reverse_lazy from django.urls import reverse, reverse_lazy
from django.views.generic import DetailView, ListView, FormView, UpdateView from django.views.generic import DetailView, ListView, UpdateView
from django.forms.models import model_to_dict from django.forms.models import model_to_dict
from django.forms import HiddenInput, CheckboxInput from django.forms import HiddenInput, CheckboxInput
from django.conf import settings from django.conf import settings
@ -42,13 +42,14 @@ from common.models import InvenTreeSetting
from company.models import SupplierPart from company.models import SupplierPart
from common.files import FileManager from common.files import FileManager
from common.views import FileManagementFormView, FileManagementAjaxView from common.views import FileManagementFormView, FileManagementAjaxView
from common.forms import UploadFileForm, MatchFieldForm
from stock.models import StockLocation from stock.models import StockLocation
import common.settings as inventree_settings import common.settings as inventree_settings
from . import forms as part_forms from . import forms as part_forms
from .bom import MakeBomTemplate, BomUploadManager, ExportBom, IsValidBOMFormat from .bom import MakeBomTemplate, ExportBom, IsValidBOMFormat
from order.models import PurchaseOrderLineItem from order.models import PurchaseOrderLineItem
from .admin import PartResource from .admin import PartResource
@ -1245,7 +1246,7 @@ class BomValidate(AjaxUpdateView):
} }
class BomUpload(InvenTreeRoleMixin, FormView): class BomUpload(InvenTreeRoleMixin, FileManagementFormView):
""" View for uploading a BOM file, and handling BOM data importing. """ View for uploading a BOM file, and handling BOM data importing.
The BOM upload process is as follows: The BOM upload process is as follows:
@ -1272,184 +1273,116 @@ class BomUpload(InvenTreeRoleMixin, FormView):
During these steps, data are passed between the server/client as JSON objects. During these steps, data are passed between the server/client as JSON objects.
""" """
template_name = 'part/bom_upload/upload_file.html'
# Context data passed to the forms (initially empty, extracted from uploaded file)
bom_headers = []
bom_columns = []
bom_rows = []
missing_columns = []
allowed_parts = []
role_required = ('part.change', 'part.add') role_required = ('part.change', 'part.add')
def get_success_url(self): class BomFileManager(FileManager):
part = self.get_object() # Fields which are absolutely necessary for valid upload
return reverse('upload-bom', kwargs={'pk': part.id}) REQUIRED_HEADERS = [
'Quantity'
]
def get_form_class(self): # Fields which are used for part matching (only one of them is needed)
ITEM_MATCH_HEADERS = [
'Part_Name',
'Part_IPN',
'Part_ID',
]
# Default form is the starting point # Fields which would be helpful but are not required
return part_forms.BomUploadSelectFile OPTIONAL_HEADERS = [
'Reference',
'Note',
'Overage',
]
def get_context_data(self, *args, **kwargs): EDITABLE_HEADERS = [
'Reference',
'Note',
'Overage'
]
ctx = super().get_context_data(*args, **kwargs) name = 'order'
form_list = [
('upload', UploadFileForm),
('fields', MatchFieldForm),
('items', part_forms.BomMatchItemForm),
]
form_steps_template = [
'part/bom_upload/upload_file.html',
'part/bom_upload/match_fields.html',
'part/bom_upload/match_parts.html',
]
form_steps_description = [
_("Upload File"),
_("Match Fields"),
_("Match Parts"),
]
form_field_map = {
'item_select': 'part',
'quantity': 'quantity',
'overage': 'overage',
'reference': 'reference',
'note': 'note',
}
file_manager_class = BomFileManager
# Give each row item access to the column it is in def get_part(self):
# This provides for much simpler template rendering """ Get part or return 404 """
rows = [] return get_object_or_404(Part, pk=self.kwargs['pk'])
for row in self.bom_rows:
row_data = row['data']
data = [] def get_context_data(self, form, **kwargs):
""" Handle context data for order """
for idx, item in enumerate(row_data): context = super().get_context_data(form=form, **kwargs)
data.append({ part = self.get_part()
'cell': item,
'idx': idx,
'column': self.bom_columns[idx]
})
rows.append({ context.update({'part': part})
'index': row.get('index', -1),
'data': data,
'part_match': row.get('part_match', None),
'part_options': row.get('part_options', self.allowed_parts),
# User-input (passed between client and server) return context
'quantity': row.get('quantity', None),
'description': row.get('description', ''),
'part_name': row.get('part_name', ''),
'part': row.get('part', None),
'reference': row.get('reference', ''),
'notes': row.get('notes', ''),
'errors': row.get('errors', ''),
})
ctx['part'] = self.part def get_allowed_parts(self):
ctx['bom_headers'] = BomUploadManager.HEADERS
ctx['bom_columns'] = self.bom_columns
ctx['bom_rows'] = rows
ctx['missing_columns'] = self.missing_columns
ctx['allowed_parts_list'] = self.allowed_parts
return ctx
def getAllowedParts(self):
""" Return a queryset of parts which are allowed to be added to this BOM. """ Return a queryset of parts which are allowed to be added to this BOM.
""" """
return self.part.get_allowed_bom_items() return self.get_part().get_allowed_bom_items()
def get(self, request, *args, **kwargs): def get_field_selection(self):
""" Perform the initial 'GET' request.
Initially returns a form for file upload """
self.request = request
# A valid Part object must be supplied. This is the 'parent' part for the BOM
self.part = get_object_or_404(Part, pk=self.kwargs['pk'])
self.form = self.get_form()
form_class = self.get_form_class()
form = self.get_form(form_class)
return self.render_to_response(self.get_context_data(form=form))
def handleBomFileUpload(self):
""" Process a BOM file upload form.
This function validates that the uploaded file was valid,
and contains tabulated data that can be extracted.
If the file does not satisfy these requirements,
the "upload file" form is again shown to the user.
"""
bom_file = self.request.FILES.get('bom_file', None)
manager = None
bom_file_valid = False
if bom_file is None:
self.form.add_error('bom_file', _('No BOM file provided'))
else:
# Create a BomUploadManager object - will perform initial data validation
# (and raise a ValidationError if there is something wrong with the file)
try:
manager = BomUploadManager(bom_file)
bom_file_valid = True
except ValidationError as e:
errors = e.error_dict
for k, v in errors.items():
self.form.add_error(k, v)
if bom_file_valid:
# BOM file is valid? Proceed to the next step!
form = None
self.template_name = 'part/bom_upload/select_fields.html'
self.extractDataFromFile(manager)
else:
form = self.form
return self.render_to_response(self.get_context_data(form=form))
def getColumnIndex(self, name):
""" Return the index of the column with the given name.
It named column is not found, return -1
"""
try:
idx = list(self.column_selections.values()).index(name)
except ValueError:
idx = -1
return idx
def preFillSelections(self):
""" Once data columns have been selected, attempt to pre-select the proper data from the database. """ Once data columns have been selected, attempt to pre-select the proper data from the database.
This function is called once the field selection has been validated. This function is called once the field selection has been validated.
The pre-fill data are then passed through to the part selection form. The pre-fill data are then passed through to the part selection form.
""" """
self.allowed_items = self.get_allowed_parts()
# Fields prefixed with "Part_" can be used to do "smart matching" against Part objects in the database # Fields prefixed with "Part_" can be used to do "smart matching" against Part objects in the database
k_idx = self.getColumnIndex('Part_ID') k_idx = self.get_column_index('Part_ID')
p_idx = self.getColumnIndex('Part_Name') p_idx = self.get_column_index('Part_Name')
i_idx = self.getColumnIndex('Part_IPN') i_idx = self.get_column_index('Part_IPN')
q_idx = self.getColumnIndex('Quantity') q_idx = self.get_column_index('Quantity')
r_idx = self.getColumnIndex('Reference') r_idx = self.get_column_index('Reference')
o_idx = self.getColumnIndex('Overage') o_idx = self.get_column_index('Overage')
n_idx = self.getColumnIndex('Note') n_idx = self.get_column_index('Note')
for row in self.bom_rows: for row in self.rows:
""" """
Iterate through each row in the uploaded data, Iterate through each row in the uploaded data,
and see if we can match the row to a "Part" object in the database. and see if we can match the row to a "Part" object in the database.
There are three potential ways to match, based on the uploaded data: There are three potential ways to match, based on the uploaded data:
a) Use the PK (primary key) field for the part, uploaded in the "Part_ID" field a) Use the PK (primary key) field for the part, uploaded in the "Part_ID" field
b) Use the IPN (internal part number) field for the part, uploaded in the "Part_IPN" field b) Use the IPN (internal part number) field for the part, uploaded in the "Part_IPN" field
c) Use the name of the part, uploaded in the "Part_Name" field c) Use the name of the part, uploaded in the "Part_Name" field
Notes: Notes:
- If using the Part_ID field, we can do an exact match against the PK field - If using the Part_ID field, we can do an exact match against the PK field
- If using the Part_IPN field, we can do an exact match against the IPN field - If using the Part_IPN field, we can do an exact match against the IPN field
- If using the Part_Name field, we can use fuzzy string matching to match "close" values - If using the Part_Name field, we can use fuzzy string matching to match "close" values
We also extract other information from the row, for the other non-matched fields: We also extract other information from the row, for the other non-matched fields:
- Quantity - Quantity
- Reference - Reference
- Overage - Overage
- Note - Note
""" """
# Initially use a quantity of zero # Initially use a quantity of zero
@ -1459,42 +1392,55 @@ class BomUpload(InvenTreeRoleMixin, FormView):
exact_match_part = None exact_match_part = None
# A list of potential Part matches # A list of potential Part matches
part_options = self.allowed_parts part_options = self.allowed_items
# Check if there is a column corresponding to "quantity" # Check if there is a column corresponding to "quantity"
if q_idx >= 0: if q_idx >= 0:
q_val = row['data'][q_idx] q_val = row['data'][q_idx]['cell']
if q_val: if q_val:
# Delete commas
q_val = q_val.replace(',', '')
try: try:
# Attempt to extract a valid quantity from the field # Attempt to extract a valid quantity from the field
quantity = Decimal(q_val) quantity = Decimal(q_val)
# Store the 'quantity' value
row['quantity'] = quantity
except (ValueError, InvalidOperation): except (ValueError, InvalidOperation):
pass pass
# Store the 'quantity' value
row['quantity'] = quantity
# Check if there is a column corresponding to "PK" # Check if there is a column corresponding to "PK"
if k_idx >= 0: if k_idx >= 0:
pk = row['data'][k_idx] pk = row['data'][k_idx]['cell']
if pk: if pk:
try: try:
# Attempt Part lookup based on PK value # Attempt Part lookup based on PK value
exact_match_part = Part.objects.get(pk=pk) exact_match_part = self.allowed_items.get(pk=pk)
except (ValueError, Part.DoesNotExist): except (ValueError, Part.DoesNotExist):
exact_match_part = None exact_match_part = None
# Check if there is a column corresponding to "Part Name" # Check if there is a column corresponding to "Part IPN" and no exact match found yet
if p_idx >= 0: if i_idx >= 0 and not exact_match_part:
part_name = row['data'][p_idx] part_ipn = row['data'][i_idx]['cell']
if part_ipn:
part_matches = [part for part in self.allowed_items if part.IPN and part_ipn.lower() == str(part.IPN.lower())]
# Check for single match
if len(part_matches) == 1:
exact_match_part = part_matches[0]
# Check if there is a column corresponding to "Part Name" and no exact match found yet
if p_idx >= 0 and not exact_match_part:
part_name = row['data'][p_idx]['cell']
row['part_name'] = part_name row['part_name'] = part_name
matches = [] matches = []
for part in self.allowed_parts: for part in self.allowed_items:
ratio = fuzz.partial_ratio(part.name + part.description, part_name) ratio = fuzz.partial_ratio(part.name + part.description, part_name)
matches.append({'part': part, 'match': ratio}) matches.append({'part': part, 'match': ratio})
@ -1503,390 +1449,67 @@ class BomUpload(InvenTreeRoleMixin, FormView):
matches = sorted(matches, key=lambda item: item['match'], reverse=True) matches = sorted(matches, key=lambda item: item['match'], reverse=True)
part_options = [m['part'] for m in matches] part_options = [m['part'] for m in matches]
# Supply list of part options for each row, sorted by how closely they match the part name
row['item_options'] = part_options
# Check if there is a column corresponding to "Part IPN" # Unless found, the 'item_match' is blank
if i_idx >= 0: row['item_match'] = None
row['part_ipn'] = row['data'][i_idx]
if exact_match_part:
# If there is an exact match based on PK or IPN, use that
row['item_match'] = exact_match_part
# Check if there is a column corresponding to "Overage" field # Check if there is a column corresponding to "Overage" field
if o_idx >= 0: if o_idx >= 0:
row['overage'] = row['data'][o_idx] row['overage'] = row['data'][o_idx]['cell']
# Check if there is a column corresponding to "Reference" field # Check if there is a column corresponding to "Reference" field
if r_idx >= 0: if r_idx >= 0:
row['reference'] = row['data'][r_idx] row['reference'] = row['data'][r_idx]['cell']
# Check if there is a column corresponding to "Note" field # Check if there is a column corresponding to "Note" field
if n_idx >= 0: if n_idx >= 0:
row['note'] = row['data'][n_idx] row['note'] = row['data'][n_idx]['cell']
# Supply list of part options for each row, sorted by how closely they match the part name def done(self, form_list, **kwargs):
row['part_options'] = part_options """ Once all the data is in, process it to add BomItem instances to the part """
# Unless found, the 'part_match' is blank self.part = self.get_part()
row['part_match'] = None items = self.get_clean_items()
if exact_match_part: # Clear BOM
# If there is an exact match based on PK, use that self.part.clear_bom()
row['part_match'] = exact_match_part
else: # Generate new BOM items
# Otherwise, check to see if there is a matching IPN for bom_item in items.values():
try: try:
if row['part_ipn']: part = Part.objects.get(pk=int(bom_item.get('part')))
part_matches = [part for part in self.allowed_parts if part.IPN and row['part_ipn'].lower() == str(part.IPN.lower())] except (ValueError, Part.DoesNotExist):
continue
# Check for single match
if len(part_matches) == 1: quantity = bom_item.get('quantity')
row['part_match'] = part_matches[0] overage = bom_item.get('overage', '')
reference = bom_item.get('reference', '')
continue note = bom_item.get('note', '')
except KeyError:
pass # Create a new BOM item
item = BomItem(
def extractDataFromFile(self, bom): part=self.part,
""" Read data from the BOM file """ sub_part=part,
quantity=quantity,
self.bom_columns = bom.columns() overage=overage,
self.bom_rows = bom.rows() reference=reference,
note=note,
def getTableDataFromPost(self): )
""" Extract table cell data from POST request.
These data are used to maintain state between sessions. try:
Table data keys are as follows:
col_name_<idx> - Column name at idx as provided in the uploaded file
col_guess_<idx> - Column guess at idx as selected in the BOM
row_<x>_col<y> - Cell data as provided in the uploaded file
"""
# Map the columns
self.column_names = {}
self.column_selections = {}
self.row_data = {}
for item in self.request.POST:
value = self.request.POST[item]
# Column names as passed as col_name_<idx> where idx is an integer
# Extract the column names
if item.startswith('col_name_'):
try:
col_id = int(item.replace('col_name_', ''))
except ValueError:
continue
col_name = value
self.column_names[col_id] = col_name
# Extract the column selections (in the 'select fields' view)
if item.startswith('col_guess_'):
try:
col_id = int(item.replace('col_guess_', ''))
except ValueError:
continue
col_name = value
self.column_selections[col_id] = value
# Extract the row data
if item.startswith('row_'):
# Item should be of the format row_<r>_col_<c>
s = item.split('_')
if len(s) < 4:
continue
# Ignore row/col IDs which are not correct numeric values
try:
row_id = int(s[1])
col_id = int(s[3])
except ValueError:
continue
if row_id not in self.row_data:
self.row_data[row_id] = {}
self.row_data[row_id][col_id] = value
self.col_ids = sorted(self.column_names.keys())
# Re-construct the data table
self.bom_rows = []
for row_idx in sorted(self.row_data.keys()):
row = self.row_data[row_idx]
items = []
for col_idx in sorted(row.keys()):
value = row[col_idx]
items.append(value)
self.bom_rows.append({
'index': row_idx,
'data': items,
'errors': {},
})
# Construct the column data
self.bom_columns = []
# Track any duplicate column selections
self.duplicates = False
for col in self.col_ids:
if col in self.column_selections:
guess = self.column_selections[col]
else:
guess = None
header = ({
'name': self.column_names[col],
'guess': guess
})
if guess:
n = list(self.column_selections.values()).count(self.column_selections[col])
if n > 1:
header['duplicate'] = True
self.duplicates = True
self.bom_columns.append(header)
# Are there any missing columns?
self.missing_columns = []
# Check that all required fields are present
for col in BomUploadManager.REQUIRED_HEADERS:
if col not in self.column_selections.values():
self.missing_columns.append(col)
# Check that at least one of the part match field is present
part_match_found = False
for col in BomUploadManager.PART_MATCH_HEADERS:
if col in self.column_selections.values():
part_match_found = True
break
# If not, notify user
if not part_match_found:
for col in BomUploadManager.PART_MATCH_HEADERS:
self.missing_columns.append(col)
def handleFieldSelection(self):
""" Handle the output of the field selection form.
Here the user is presented with the raw data and must select the
column names and which rows to process.
"""
# Extract POST data
self.getTableDataFromPost()
valid = len(self.missing_columns) == 0 and not self.duplicates
if valid:
# Try to extract meaningful data
self.preFillSelections()
self.template_name = 'part/bom_upload/select_parts.html'
else:
self.template_name = 'part/bom_upload/select_fields.html'
return self.render_to_response(self.get_context_data(form=None))
def handlePartSelection(self):
# Extract basic table data from POST request
self.getTableDataFromPost()
# Keep track of the parts that have been selected
parts = {}
# Extract other data (part selections, etc)
for key in self.request.POST:
value = self.request.POST[key]
# Extract quantity from each row
if key.startswith('quantity_'):
try:
row_id = int(key.replace('quantity_', ''))
row = self.getRowByIndex(row_id)
if row is None:
continue
q = Decimal(1)
try:
q = Decimal(value)
if q < 0:
row['errors']['quantity'] = _('Quantity must be greater than zero')
if 'part' in row.keys():
if row['part'].trackable:
# Trackable parts must use integer quantities
if not q == int(q):
row['errors']['quantity'] = _('Quantity must be integer value for trackable parts')
except (ValueError, InvalidOperation):
row['errors']['quantity'] = _('Enter a valid quantity')
row['quantity'] = q
except ValueError:
continue
# Extract part from each row
if key.startswith('part_'):
try:
row_id = int(key.replace('part_', ''))
row = self.getRowByIndex(row_id)
if row is None:
continue
except ValueError:
# Row ID non integer value
continue
try:
part_id = int(value)
part = Part.objects.get(id=part_id)
except ValueError:
row['errors']['part'] = _('Select valid part')
continue
except Part.DoesNotExist:
row['errors']['part'] = _('Select valid part')
continue
# Keep track of how many of each part we have seen
if part_id in parts:
parts[part_id]['quantity'] += 1
row['errors']['part'] = _('Duplicate part selected')
else:
parts[part_id] = {
'part': part,
'quantity': 1,
}
row['part'] = part
if part.trackable:
# For trackable parts, ensure the quantity is an integer value!
if 'quantity' in row.keys():
q = row['quantity']
if not q == int(q):
row['errors']['quantity'] = _('Quantity must be integer value for trackable parts')
# Extract other fields which do not require further validation
for field in ['reference', 'notes']:
if key.startswith(field + '_'):
try:
row_id = int(key.replace(field + '_', ''))
row = self.getRowByIndex(row_id)
if row:
row[field] = value
except:
continue
# Are there any errors after form handling?
valid = True
for row in self.bom_rows:
# Has a part been selected for the given row?
part = row.get('part', None)
if part is None:
row['errors']['part'] = _('Select a part')
else:
# Will the selected part result in a recursive BOM?
try:
part.checkAddToBOM(self.part)
except ValidationError:
row['errors']['part'] = _('Selected part creates a circular BOM')
# Has a quantity been specified?
if row.get('quantity', None) is None:
row['errors']['quantity'] = _('Specify quantity')
errors = row.get('errors', [])
if len(errors) > 0:
valid = False
self.template_name = 'part/bom_upload/select_parts.html'
ctx = self.get_context_data(form=None)
if valid:
self.part.clear_bom()
# Generate new BOM items
for row in self.bom_rows:
part = row.get('part')
quantity = row.get('quantity')
reference = row.get('reference', '')
notes = row.get('notes', '')
# Create a new BOM item!
item = BomItem(
part=self.part,
sub_part=part,
quantity=quantity,
reference=reference,
note=notes
)
item.save() item.save()
except IntegrityError:
# BomItem already exists
pass
# Redirect to the BOM view return HttpResponseRedirect(reverse('part-bom', kwargs={'pk': self.kwargs['pk']}))
return HttpResponseRedirect(reverse('part-bom', kwargs={'pk': self.part.id}))
else:
ctx['form_errors'] = True
return self.render_to_response(ctx)
def getRowByIndex(self, idx):
for row in self.bom_rows:
if row['index'] == idx:
return row
return None
def post(self, request, *args, **kwargs):
""" Perform the various 'POST' requests required.
"""
self.request = request
self.part = get_object_or_404(Part, pk=self.kwargs['pk'])
self.allowed_parts = self.getAllowedParts()
self.form = self.get_form(self.get_form_class())
# Did the user POST a file named bom_file?
form_step = request.POST.get('form_step', None)
if form_step == 'select_file':
return self.handleBomFileUpload()
elif form_step == 'select_fields':
return self.handleFieldSelection()
elif form_step == 'select_parts':
return self.handlePartSelection()
return self.render_to_response(self.get_context_data(form=self.form))
class PartExport(AjaxView): class PartExport(AjaxView):