Mirrored_Repos/InvenTree
1
0
Fork 0
mirror of https://github.com/inventree/InvenTree synced 2024-08-30 18:33:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major improvements to BOM upload

Browse Source 
- Column headings now match the values provided in BOM import template
- Add a new column for part selection, while displaying all imported data
- Better code documentation
- Improve data validation
- Allow decimal quantity (not just integer!)
- Better matching logic
This commit is contained in:
Oliver Walters 2020-08-18 14:01:01 +10:00
parent 7349b396ca
commit 68fb599c73
4 changed files with 157 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
InvenTree/part/bom.py
View File

@ -102,26 +102,23 @@ class BomUploadManager:
# Fields which are absolutely necessary for valid upload
REQUIRED_HEADERS = [
'Part',
'Part_Name',
'Quantity'
]
# Fields which would be helpful but are not required
OPTIONAL_HEADERS = [
'Part_IPN',
'Part_ID',
'Reference',
'Notes',
'Note',
'Overage',
'Description',
'Category',
'Supplier',
'Manufacturer',
'MPN',
'IPN',
]
EDITABLE_HEADERS = [
'Reference',
'Notes'
'Note',
'Overage'
]
HEADERS = REQUIRED_HEADERS + OPTIONAL_HEADERS
@ -171,6 +168,11 @@ class BomUploadManager:
if h.lower() == header.lower():
return h
# Try for a case-insensitive match with space replacement
for h in self.HEADERS:
if h.lower() == header.lower().replace(' ', '_'):
return h
# Finally, look for a close match using fuzzy matching
matches = []

13
InvenTree/part/templates/part/bom_upload/select_fields.html
View File

@ -1,17 +1,18 @@
{% extends "part/part_base.html" %}
{% load static %}
{% load i18n %}
{% load inventree_extras %}
{% block details %}
{% include "part/tabs.html" with tab='bom' %}
<h4>Upload Bill of Materials</h4>
<h4>{% trans "Upload Bill of Materials" %}</h4>
<p>Step 2 - Select Fields</p>
<p>{% trans "Step 2 - Select Fields" %}</p>
<hr>
{% if missing_columns and missing_columns|length > 0 %}
<div class='alert alert-danger alert-block' role='alert'>
Missing selections for the following required columns:
{% trans "Missing selections for the following required columns" %}:
<br>
<ul>
{% for col in missing_columns %}
@ -22,7 +23,7 @@
{% endif %}
<form method="post" action='' class='js-modal-form' enctype="multipart/form-data">
<button type="submit" class="save btn btn-default">Submit Selections</button>
<button type="submit" class="save btn btn-default">{% trans "Submit Selections" %}</button>
{% csrf_token %}
<input type='hidden' name='form_step' value='select_fields'/>
@ -31,7 +32,7 @@
<thead>
<tr>
<th></th>
<th>Row</th>
<th>{% trans "File Fields" %}</th>
{% for col in bom_columns %}
<th>
<div>
@ -48,7 +49,7 @@
<tbody>
<tr>
<td></td>
<td></td>
<td>{% trans "Match Fields" %}</td>
{% for col in bom_columns %}
<td>
<select class='select' id='id_col_{{ forloop.counter0 }}' name='col_guess_{{ forloop.counter0 }}'>

50
InvenTree/part/templates/part/bom_upload/select_parts.html
View File

@ -1,23 +1,24 @@
{% extends "part/part_base.html" %}
{% load static %}
{% load i18n %}
{% load inventree_extras %}
{% block details %}
{% include "part/tabs.html" with tab="bom" %}
<h4>Upload Bill of Materials</h4>
<h4>{% trans "Upload Bill of Materials" %}</h4>
<p>Step 3 - Select Parts</p>
<p>{% trans "Step 3 - Select Parts" %}</p>
<hr>
{% if form_errors %}
<div class='alert alert-danger alert-block' role='alert'>
Errors exist in the submitted data.
{% trans "Errors exist in the submitted data" %}
</div>
{% endif %}
<form method="post" action='' class='js-modal-form' enctype="multipart/form-data">
<button type="submit" class="save btn btn-default">Submit BOM</button>
<button type="submit" class="save btn btn-default">{% trans "Submit BOM" %}</button>
{% csrf_token %}
{% load crispy_forms_tags %}
@ -29,7 +30,8 @@
<tr>
<th></th>
<th></th>
<th>Row</th>
<th>{% trans "Row" %}</th>
<th>{% trans "Select Part" %}</th>
{% for col in bom_columns %}
<th>
<input type='hidden' name='col_name_{{ forloop.counter0 }}' value='{{ col.name }}'/>
@ -51,39 +53,39 @@
<span row_id='{{ forloop.counter }}' class='fas fa-trash-alt icon-red'></span>
</button>
</td>
<td>
</td>
<td></td>
<td>{% add row.index 1 %}</td>
<td>
<button class='btn btn-default btn-create' onClick='newPartFromBomWizard()' id='new_part_row_{{ row.index }}' title='Create new part' type='button'>
<span row_id='{{ row.index }}' class='fas fa-plus icon-green'/>
</button>
<select class='select bomselect' id='select_part_{{ row.index }}' name='part_{{ row.index }}'>
<option value=''>--- {% trans "Select Part" %} ---</option>
{% for part in row.part_options %}
<option value='{{ part.id }}' {% if part.id == row.part.id %} selected='selected' {% elif part.id == row.part_match.id %} selected='selected' {% endif %}>
{{ part.full_name }} - {{ part.description }}
</option>
{% endfor %}
</select>
{% if row.errors.part %}
<p class='help-inline'>{{ row.errors.part }}</p>
{% endif %}
</td>
{% for item in row.data %}
<td>
{% if item.column.guess == 'Part' %}
<button class='btn btn-default btn-create' onClick='newPartFromBomWizard()' id='new_part_row_{{ row.index }}' title='Create new part' type='button'>
<span row_id='{{ row.index }}' class='fas fa-trash-alt icon-red'/>
</button>
<select class='select bomselect' id='select_part_{{ row.index }}' name='part_{{ row.index }}'>
{% if row.part_match %}
{% with row.part_match as part %}
<option value='{{ part.id }}'{% if part.id == row.part.id %} selected='selected'{% endif %}>{{ part.full_name }} - {{ part.description }}</option>
{% endwith %}
{% endif %}
<option value=''>---------</option>
{% for part in row.part_options %}
<option value='{{ part.id }}'{% if part.id == row.part.id %} selected='selected'{% endif %}>{{ part.full_name }} - {{ part.description }}</option>
{% endfor %}
</select>
<i>{{ item.cell }}</i>
{% if row.errors.part %}
<p class='help-inline'>{{ row.errors.part }}</p>
{% endif %}
{% elif item.column.guess == 'Quantity' %}
<input name='quantity_{{ row.index }}' class='numberinput' type='number' min='1' value='{{ row.quantity }}'/>
<input name='quantity_{{ row.index }}' class='numberinput' type='number' min='1' step='any' value='{% decimal row.quantity %}'/>
{% if row.errors.quantity %}
<p class='help-inline'>{{ row.errors.quantity }}</p>
{% endif %}
{% elif item.column.guess == 'Reference' %}
<input name='reference_{{ row.index }}' value='{{ row.reference }}'/>
{% elif item.column.guess == 'Notes' %}
{% elif item.column.guess == 'Note' %}
<input name='notes_{{ row.index }}' value='{{ row.notes }}'/>
{% elif item.column.guess == 'Overage' %}
<input name='overage_{{ row.index }}' value='{{ row.overage }}'/>

158
InvenTree/part/views.py
View File

@ -19,7 +19,7 @@ from django.conf import settings
import os
from rapidfuzz import fuzz
from decimal import Decimal
from decimal import Decimal, InvalidOperation
from .models import PartCategory, Part, PartAttachment
from .models import PartParameterTemplate, PartParameter
@ -948,81 +948,133 @@ class BomUpload(FormView):
The pre-fill data are then passed through to the part selection form.
"""
# Fields prefixed with "Part_" can be used to do "smart matching" against Part objects in the database
k_idx = self.getColumnIndex('Part_ID')
p_idx = self.getColumnIndex('Part_Name')
i_idx = self.getColumnIndex('Part_IPN')
q_idx = self.getColumnIndex('Quantity')
p_idx = self.getColumnIndex('Part')
i_idx = self.getColumnIndex('IPN')
d_idx = self.getColumnIndex('Description')
r_idx = self.getColumnIndex('Reference')
n_idx = self.getColumnIndex('Notes')
o_idx = self.getColumnIndex('Overage')
n_idx = self.getColumnIndex('Note')
for row in self.bom_rows:
"""
quantity = 0
part = None
Iterate through each row in the uploaded data,
and see if we can match the row to a "Part" object in the database.
There are three potential ways to match, based on the uploaded data:
a) Use the PK (primary key) field for the part, uploaded in the "Part_ID" field
b) Use the IPN (internal part number) field for the part, uploaded in the "Part_IPN" field
c) Use the name of the part, uploaded in the "Part_Name" field
Notes:
- If using the Part_ID field, we can do an exact match against the PK field
- If using the Part_IPN field, we can do an exact match against the IPN field
- If using the Part_Name field, we can use fuzzy string matching to match "close" values
We also extract other information from the row, for the other non-matched fields:
- Quantity
- Reference
- Overage
- Note
"""
# Initially use a quantity of zero
quantity = Decimal(0)
# Initially we do not have a part to reference
exact_match_part = None
# A list of potential Part matches
part_options = self.allowed_parts
# Check if there is a column corresponding to "quantity"
if q_idx >= 0:
q_val = row['data'][q_idx]
try:
quantity = int(q_val)
except ValueError:
pass
if q_val:
try:
# Attempt to extract a valid quantity from the field
quantity = Decimal(q_val)
except (ValueError, InvalidOperation):
pass
# Store the 'quantity' value
row['quantity'] = quantity
# Check if there is a column corresponding to "PK"
if k_idx >= 0:
pk = row['data'][k_idx]
if pk:
try:
# Attempt Part lookup based on PK value
exact_match_part = Part.objects.get(pk=pk)
except (ValueError, Part.DoesNotExist):
exact_match_part = None
# Check if there is a column corresponding to "Part Name"
if p_idx >= 0:
part_name = row['data'][p_idx]
row['part_name'] = part_name
# Fuzzy match the values and see what happens
matches = []
for part in self.allowed_parts:
ratio = fuzz.partial_ratio(part.name + part.description, part_name)
matches.append({'part': part, 'match': ratio})
# Sort matches by the 'strength' of the match ratio
if len(matches) > 0:
matches = sorted(matches, key=lambda item: item['match'], reverse=True)
part_options = [m['part'] for m in matches]
# Check if there is a column corresponding to "Part IPN"
if i_idx >= 0:
row['part_ipn'] = row['data'][i_idx]
if d_idx >= 0:
row['description'] = row['data'][d_idx]
# Check if there is a column corresponding to "Overage" field
if o_idx >= 0:
row['overage'] = row['data'][o_idx]
# Check if there is a column corresponding to "Reference" field
if r_idx >= 0:
row['reference'] = row['data'][r_idx]
# Check if there is a column corresponding to "Note" field
if n_idx >= 0:
row['notes'] = row['data'][n_idx]
row['note'] = row['data'][n_idx]
# Supply list of part options for each row, sorted by how closely they match the part name
row['part_options'] = part_options
row['quantity'] = quantity
# Unless found, the 'part_match' is blank
row['part_match'] = None
# Part selection using IPN
try:
if row['part_ipn']:
part_matches = [part for part in self.allowed_parts if row['part_ipn'] == part.IPN]
part_options = [part for part in self.allowed_parts if part not in part_matches]
# Check for single match
if len(part_matches) == 1:
row['part_match'] = part_matches[0]
row['part_options'] = part_options
continue
except KeyError:
pass
# Part selection using Part Name
match_limit = 100
part_matches = [m['part'] for m in matches if m['match'] >= match_limit]
# Check for single match
if len(part_matches) == 1:
row['part_match'] = part_matches[0]
row['part_options'] = [m['part'] for m in matches if m['match'] < match_limit]
if exact_match_part:
# If there is an exact match based on PK, use that
row['part_match'] = exact_match_part
else:
row['part_options'] = [m['part'] for m in matches]
# Otherwise, check to see if there is a matching IPN
try:
if row['part_ipn']:
part_matches = [part for part in self.allowed_parts if row['part_ipn'].lower() == part.IPN.lower()]
# Check for single match
if len(part_matches) == 1:
row['part_match'] = part_matches[0]
continue
except KeyError:
pass
print(row, row['part_match'], len(row['part_options']))
def extractDataFromFile(self, bom):
""" Read data from the BOM file """
@ -1190,13 +1242,20 @@ class BomUpload(FormView):
if row is None:
continue
q = 1
q = Decimal(1)
try:
q = int(value)
if q <= 0:
q = Decimal(value)
if q < 0:
row['errors']['quantity'] = _('Quantity must be greater than zero')
except ValueError:
if 'part' in row.keys():
if row['part'].trackable:
# Trackable parts must use integer quantities
if not q == int(q):
row['errors']['quantity'] = _('Quantity must be integer value for trackable parts')
except (ValueError, InvalidOperation):
row['errors']['quantity'] = _('Enter a valid quantity')
row['quantity'] = q
@ -1206,6 +1265,7 @@ class BomUpload(FormView):
# Extract part from each row
if key.startswith('part_'):
try:
row_id = int(key.replace('part_', ''))
@ -1239,6 +1299,14 @@ class BomUpload(FormView):
row['part'] = part
if part.trackable:
# For trackable parts, ensure the quantity is an integer value!
if 'quantity' in row.keys():
q = row['quantity']
if not q == int(q):
row['errors']['quantity'] = _('Quantity must be integer value for trackable parts')
# Extract other fields which do not require further validation
for field in ['reference', 'notes']:
if key.startswith(field + '_'):