diff --git a/src/backend/InvenTree/InvenTree/api_version.py b/src/backend/InvenTree/InvenTree/api_version.py index 3a2894e99d..8494ebb26d 100644 --- a/src/backend/InvenTree/InvenTree/api_version.py +++ b/src/backend/InvenTree/InvenTree/api_version.py @@ -1,12 +1,15 @@ """InvenTree API version information.""" # InvenTree API version -INVENTREE_API_VERSION = 221 +INVENTREE_API_VERSION = 222 """Increment this API version number whenever there is a significant change to the API that any clients need to know about.""" INVENTREE_API_TEXT = """ +v222 - 2024-07-14 : https://github.com/inventree/InvenTree/pull/7635 + - Adjust the BomItem API endpoint to improve data import process + v221 - 2024-07-13 : https://github.com/inventree/InvenTree/pull/7636 - Adds missing fields from StockItemBriefSerializer - Adds missing fields from PartBriefSerializer diff --git a/src/backend/InvenTree/importer/migrations/0002_dataimportsession_field_overrides.py b/src/backend/InvenTree/importer/migrations/0002_dataimportsession_field_overrides.py new file mode 100644 index 0000000000..9d00ce956b --- /dev/null +++ b/src/backend/InvenTree/importer/migrations/0002_dataimportsession_field_overrides.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.14 on 2024-07-12 03:35 + +from django.db import migrations, models +import importer.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('importer', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='dataimportsession', + name='field_overrides', + field=models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Overrides'), + ), + ] diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 3eb811c262..83c417f782 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -1,5 +1,6 @@ """Model definitions for the 'importer' app.""" +import json import logging from django.contrib.auth.models import User @@ -32,6 +33,7 @@ class DataImportSession(models.Model): status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import field_defaults: JSONField for field default values + field_overrides: JSONField for field override values """ @staticmethod @@ -92,6 +94,13 @@ class DataImportSession(models.Model): validators=[importer.validators.validate_field_defaults], ) + field_overrides = models.JSONField( + blank=True, + null=True, + verbose_name=_('Field Overrides'), + validators=[importer.validators.validate_field_defaults], + ) + @property def field_mapping(self): """Construct a dict of field mappings for this import session. @@ -132,8 +141,15 @@ class DataImportSession(models.Model): matched_columns = set() + field_overrides = self.field_overrides or {} + # Create a default mapping for each available field in the database for field, field_def in serializer_fields.items(): + # If an override value is provided for the field, + # skip creating a mapping for this field + if field in field_overrides: + continue + # Generate a list of possible column names for this field field_options = [ field, @@ -181,10 +197,15 @@ class DataImportSession(models.Model): required_fields = self.required_fields() field_defaults = self.field_defaults or {} + field_overrides = self.field_overrides or {} missing_fields = [] for field in required_fields.keys(): + # An override value exists + if field in field_overrides: + continue + # A default value exists if field in field_defaults and field_defaults[field]: continue @@ -265,6 +286,18 @@ class DataImportSession(models.Model): self.status = DataImportStatusCode.PROCESSING.value self.save() + def check_complete(self) -> bool: + """Check if the import session is complete.""" + if self.completed_row_count < self.row_count: + return False + + # Update the status of this session + if self.status != DataImportStatusCode.COMPLETE.value: + self.status = DataImportStatusCode.COMPLETE.value + self.save() + + return True + @property def row_count(self): """Return the number of rows in the import session.""" @@ -467,6 +500,34 @@ class DataImportRow(models.Model): complete = models.BooleanField(default=False, verbose_name=_('Complete')) + @property + def default_values(self) -> dict: + """Return a dict object of the 'default' values for this row.""" + defaults = self.session.field_defaults or {} + + if type(defaults) is not dict: + try: + defaults = json.loads(str(defaults)) + except json.JSONDecodeError: + logger.warning('Failed to parse default values for import row') + defaults = {} + + return defaults + + @property + def override_values(self) -> dict: + """Return a dict object of the 'override' values for this row.""" + overrides = self.session.field_overrides or {} + + if type(overrides) is not dict: + try: + overrides = json.loads(str(overrides)) + except json.JSONDecodeError: + logger.warning('Failed to parse override values for import row') + overrides = {} + + return overrides + def extract_data( self, available_fields: dict = None, field_mapping: dict = None, commit=True ): @@ -477,14 +538,24 @@ class DataImportRow(models.Model): if not available_fields: available_fields = self.session.available_fields() - default_values = self.session.field_defaults or {} + overrride_values = self.override_values + default_values = self.default_values data = {} # We have mapped column (file) to field (serializer) already for field, col in field_mapping.items(): + # Data override (force value and skip any further checks) + if field in overrride_values: + data[field] = overrride_values[field] + continue + + # Default value (if provided) + if field in default_values: + data[field] = default_values[field] + # If this field is *not* mapped to any column, skip - if not col: + if not col or col not in self.row_data: continue # Extract field type @@ -516,11 +587,14 @@ class DataImportRow(models.Model): - If available, we use the "default" values provided by the import session - If available, we use the "override" values provided by the import session """ - data = self.session.field_defaults or {} + data = self.default_values if self.data: data.update(self.data) + # Override values take priority, if present + data.update(self.override_values) + return data def construct_serializer(self): @@ -568,6 +642,8 @@ class DataImportRow(models.Model): self.complete = True self.save() + self.session.check_complete() + except Exception as e: self.errors = {'non_field_errors': str(e)} result = False diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index 61bcb26960..2400dc179d 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -1,5 +1,7 @@ """API serializers for the importer app.""" +import json + from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ @@ -47,6 +49,7 @@ class DataImportSessionSerializer(InvenTreeModelSerializer): 'columns', 'column_mappings', 'field_defaults', + 'field_overrides', 'row_count', 'completed_row_count', ] @@ -75,6 +78,32 @@ class DataImportSessionSerializer(InvenTreeModelSerializer): user_detail = UserSerializer(source='user', read_only=True, many=False) + def validate_field_defaults(self, defaults): + """De-stringify the field defaults.""" + if defaults is None: + return None + + if type(defaults) is not dict: + try: + defaults = json.loads(str(defaults)) + except: + raise ValidationError(_('Invalid field defaults')) + + return defaults + + def validate_field_overrides(self, overrides): + """De-stringify the field overrides.""" + if overrides is None: + return None + + if type(overrides) is not dict: + try: + overrides = json.loads(str(overrides)) + except: + raise ValidationError(_('Invalid field overrides')) + + return overrides + def create(self, validated_data): """Override create method for this serializer. @@ -167,4 +196,7 @@ class DataImportAcceptRowSerializer(serializers.Serializer): for row in rows: row.validate(commit=True) + if session := self.context.get('session', None): + session.check_complete() + return rows diff --git a/src/backend/InvenTree/importer/validators.py b/src/backend/InvenTree/importer/validators.py index 34e48b1862..166c30acc6 100644 --- a/src/backend/InvenTree/importer/validators.py +++ b/src/backend/InvenTree/importer/validators.py @@ -1,6 +1,6 @@ """Custom validation routines for the 'importer' app.""" -import os +import json from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ @@ -46,4 +46,8 @@ def validate_field_defaults(value): return if type(value) is not dict: - raise ValidationError(_('Value must be a valid dictionary object')) + # OK if we can parse it as JSON + try: + value = json.loads(value) + except json.JSONDecodeError: + raise ValidationError(_('Value must be a valid dictionary object')) diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index b05a426ed1..a87453348f 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -1480,28 +1480,30 @@ class BomItemSerializer( ): """Serializer for BomItem object.""" + import_exclude_fields = ['validated', 'substitutes'] + class Meta: """Metaclass defining serializer fields.""" model = BomItem fields = [ + 'part', + 'sub_part', + 'reference', + 'quantity', + 'overage', 'allow_variants', 'inherited', - 'note', 'optional', 'consumable', - 'overage', + 'note', 'pk', - 'part', 'part_detail', 'pricing_min', 'pricing_max', 'pricing_min_total', 'pricing_max_total', 'pricing_updated', - 'quantity', - 'reference', - 'sub_part', 'sub_part_detail', 'substitutes', 'validated', diff --git a/src/frontend/src/components/forms/ApiForm.tsx b/src/frontend/src/components/forms/ApiForm.tsx index 3456b2e07b..8cc11e915c 100644 --- a/src/frontend/src/components/forms/ApiForm.tsx +++ b/src/frontend/src/components/forms/ApiForm.tsx @@ -384,21 +384,40 @@ export function ApiForm({ let method = props.method?.toLowerCase() ?? 'get'; let hasFiles = false; - mapFields(fields, (_path, field) => { - if (field.field_type === 'file upload') { - hasFiles = true; - } - }); // Optionally pre-process the data before submitting it if (props.processFormData) { data = props.processFormData(data); } + let dataForm = new FormData(); + + Object.keys(data).forEach((key: string) => { + let value: any = data[key]; + let field_type = fields[key]?.field_type; + + if (field_type == 'file upload') { + hasFiles = true; + } + + // Stringify any JSON objects + if (typeof value === 'object') { + switch (field_type) { + case 'file upload': + break; + default: + value = JSON.stringify(value); + break; + } + } + + dataForm.append(key, value); + }); + return api({ method: method, url: url, - data: data, + data: hasFiles ? dataForm : data, timeout: props.timeout, headers: { 'Content-Type': hasFiles ? 'multipart/form-data' : 'application/json' @@ -462,7 +481,11 @@ export function ApiForm({ for (const [k, v] of Object.entries(errors)) { const path = _path ? `${_path}.${k}` : k; - if (k === 'non_field_errors' || k === '__all__') { + // Determine if field "k" is valid (exists and is visible) + let field = fields[k]; + let valid = field && !field.hidden; + + if (!valid || k === 'non_field_errors' || k === '__all__') { if (Array.isArray(v)) { _nonFieldErrors.push(...v); } diff --git a/src/frontend/src/components/forms/fields/ApiFormField.tsx b/src/frontend/src/components/forms/fields/ApiFormField.tsx index 736d8cda8c..5116790fc9 100644 --- a/src/frontend/src/components/forms/fields/ApiFormField.tsx +++ b/src/frontend/src/components/forms/fields/ApiFormField.tsx @@ -21,6 +21,7 @@ import { DependentField } from './DependentField'; import { NestedObjectField } from './NestedObjectField'; import { RelatedModelField } from './RelatedModelField'; import { TableField } from './TableField'; +import TextField from './TextField'; export type ApiFormData = UseFormReturnType>; @@ -223,21 +224,11 @@ export function ApiFormField({ case 'url': case 'string': return ( - onChange(event.currentTarget.value)} - rightSection={ - value && !definition.required ? ( - onChange('')} /> - ) : null - } + ); case 'boolean': diff --git a/src/frontend/src/components/forms/fields/TextField.tsx b/src/frontend/src/components/forms/fields/TextField.tsx new file mode 100644 index 0000000000..ddb9e8843f --- /dev/null +++ b/src/frontend/src/components/forms/fields/TextField.tsx @@ -0,0 +1,66 @@ +import { TextInput } from '@mantine/core'; +import { useDebouncedValue } from '@mantine/hooks'; +import { IconX } from '@tabler/icons-react'; +import { useCallback, useEffect, useId, useState } from 'react'; +import { FieldValues, UseControllerReturn } from 'react-hook-form'; + +/* + * Custom implementation of the mantine component, + * used for rendering text input fields in forms. + * Uses a debounced value to prevent excessive re-renders. + */ +export default function TextField({ + controller, + fieldName, + definition, + onChange +}: { + controller: UseControllerReturn; + definition: any; + fieldName: string; + onChange: (value: any) => void; +}) { + const fieldId = useId(); + const { + field, + fieldState: { error } + } = controller; + + const { value } = field; + + const [rawText, setRawText] = useState(value); + const [debouncedText] = useDebouncedValue(rawText, 250); + + useEffect(() => { + setRawText(value); + }, [value]); + + const onTextChange = useCallback((value: any) => { + setRawText(value); + }, []); + + useEffect(() => { + if (debouncedText !== value) { + onChange(debouncedText); + } + }, [debouncedText]); + + return ( + onTextChange(event.currentTarget.value)} + rightSection={ + value && !definition.required ? ( + onTextChange('')} /> + ) : null + } + /> + ); +} diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 46a3378267..095a42bd59 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,5 +1,5 @@ import { t } from '@lingui/macro'; -import { Group, HoverCard, Stack, Text } from '@mantine/core'; +import { Group, HoverCard, Paper, Space, Stack, Text } from '@mantine/core'; import { notifications } from '@mantine/notifications'; import { IconArrowRight, @@ -26,6 +26,7 @@ import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; import { ActionButton } from '../buttons/ActionButton'; import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; +import { ProgressBar } from '../items/ProgressBar'; import { RenderRemoteInstance } from '../render/Instance'; function ImporterDataCell({ @@ -178,6 +179,8 @@ export default function ImporterDataSelector({ table.clearSelectedRecords(); notifications.hide('importing-rows'); table.refreshTable(); + + session.refreshSession(); }); }, [session.sessionId, table.refreshTable] @@ -191,6 +194,7 @@ export default function ImporterDataSelector({ title: t`Edit Data`, fields: selectedFields, initialData: selectedRow.data, + fetchInitialData: false, processFormData: (data: any) => { // Construct fields back into a single object return { @@ -374,6 +378,18 @@ export default function ImporterDataSelector({ {editRow.modal} {deleteRow.modal} + + + {t`Processing Data`} + + + + + { + session.refreshSession(); + } }} /> diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index 370e8da1a0..0fe47653db 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -2,19 +2,23 @@ import { t } from '@lingui/macro'; import { Alert, Button, - Divider, Group, + Paper, Select, - SimpleGrid, + Space, Stack, + Table, Text } from '@mantine/core'; +import { IconCheck } from '@tabler/icons-react'; import { useCallback, useEffect, useMemo, useState } from 'react'; import { api } from '../../App'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ImportSessionState } from '../../hooks/UseImportSession'; import { apiUrl } from '../../states/ApiState'; +import { StandaloneField } from '../forms/StandaloneField'; +import { ApiFormFieldType } from '../forms/fields/ApiFormField'; function ImporterColumn({ column, options }: { column: any; options: any[] }) { const [errorMessage, setErrorMessage] = useState(''); @@ -54,6 +58,7 @@ function ImporterColumn({ column, options }: { column: any; options: any[] }) {