Refctor image downloader (#3393)

* Adds configurable setting for maximum remote image size

* Add helper function for downloading image from remote URL

- Will replace existing function
- Performs more thorough sanity checking

* Replace existing image downloading code

- part image uses new generic function
- company image uses new generic function

* Rearrange settings

* Refactor and cleanup existing views / forms

* Add unit testing for image downloader function

* Refactor image downloader forms

- Part image download now uses the API
- Company image download now uses the API
- Remove outdated forms / views / templates

* Increment API version

* Prevent remote image download via API if the setting is not enabled

* Do not attempt to validate or extract image from blank URL

* Fix custom save() serializer methods
This commit is contained in:
Oliver 2022-07-25 11:17:59 +10:00 committed by GitHub
parent d32054b53b
commit eecb26676e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 279 additions and 301 deletions

View File

@ -2,11 +2,15 @@
# InvenTree API version
INVENTREE_API_VERSION = 65
INVENTREE_API_VERSION = 66
"""
Increment this API version number whenever there is a significant change to the API that any clients need to know about
v66 -> 2022-07-24 : https://github.com/inventree/InvenTree/pull/3393
- Part images can now be downloaded from a remote URL via the API
- Company images can now be downloaded from a remote URL via the API
v65 -> 2022-07-15 : https://github.com/inventree/InvenTree/pull/3335
- Annotates 'in_stock' quantity to the SupplierPart API

View File

@ -3,6 +3,7 @@
import io
import json
import logging
import os
import os.path
import re
from decimal import Decimal, InvalidOperation
@ -12,10 +13,12 @@ from django.conf import settings
from django.contrib.auth.models import Permission
from django.core.exceptions import FieldError, ValidationError
from django.core.files.storage import default_storage
from django.core.validators import URLValidator
from django.http import StreamingHttpResponse
from django.test import TestCase
from django.utils.translation import gettext_lazy as _
import requests
from djmoney.money import Money
from PIL import Image
@ -87,6 +90,95 @@ def construct_absolute_url(*arg):
return url
def download_image_from_url(remote_url, timeout=2.5):
"""Download an image file from a remote URL.
This is a potentially dangerous operation, so we must perform some checks:
- The remote URL is available
- The Content-Length is provided, and is not too large
- The file is a valid image file
Arguments:
remote_url: The remote URL to retrieve image
max_size: Maximum allowed image size (default = 1MB)
timeout: Connection timeout in seconds (default = 5)
Returns:
An in-memory PIL image file, if the download was successful
Raises:
requests.exceptions.ConnectionError: Connection could not be established
requests.exceptions.Timeout: Connection timed out
requests.exceptions.HTTPError: Server responded with invalid response code
ValueError: Server responded with invalid 'Content-Length' value
TypeError: Response is not a valid image
"""
# Check that the provided URL at least looks valid
validator = URLValidator()
validator(remote_url)
# Calculate maximum allowable image size (in bytes)
max_size = int(InvenTreeSetting.get_setting('INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE')) * 1024 * 1024
try:
response = requests.get(
remote_url,
timeout=timeout,
allow_redirects=True,
stream=True,
)
# Throw an error if anything goes wrong
response.raise_for_status()
except requests.exceptions.ConnectionError as exc:
raise Exception(_("Connection error") + f": {str(exc)}")
except requests.exceptions.Timeout as exc:
raise exc
except requests.exceptions.HTTPError:
raise requests.exceptions.HTTPError(_("Server responded with invalid status code") + f": {response.status_code}")
except Exception as exc:
raise Exception(_("Exception occurred") + f": {str(exc)}")
if response.status_code != 200:
raise Exception(_("Server responded with invalid status code") + f": {response.status_code}")
try:
content_length = int(response.headers.get('Content-Length', 0))
except ValueError:
raise ValueError(_("Server responded with invalid Content-Length value"))
if content_length > max_size:
raise ValueError(_("Image size is too large"))
# Download the file, ensuring we do not exceed the reported size
fo = io.BytesIO()
dl_size = 0
chunk_size = 64 * 1024
for chunk in response.iter_content(chunk_size=chunk_size):
dl_size += len(chunk)
if dl_size > max_size:
raise ValueError(_("Image download exceeded maximum size"))
fo.write(chunk)
if dl_size == 0:
raise ValueError(_("Remote server returned empty response"))
# Now, attempt to convert the downloaded data to a valid image file
# img.verify() will throw an exception if the image is not valid
try:
img = Image.open(fo).convert()
img.verify()
except Exception:
raise TypeError(_("Supplied URL is not a valid image file"))
return img
def TestIfImage(img):
"""Test if an image file is indeed an image."""
try:

View File

@ -19,6 +19,9 @@ from rest_framework.fields import empty
from rest_framework.serializers import DecimalField
from rest_framework.utils import model_meta
from common.models import InvenTreeSetting
from InvenTree.helpers import download_image_from_url
class InvenTreeMoneySerializer(MoneyField):
"""Custom serializer for 'MoneyField', which ensures that passed values are numerically valid.
@ -576,3 +579,39 @@ class DataFileExtractSerializer(serializers.Serializer):
def save(self):
"""No "save" action for this serializer."""
pass
class RemoteImageMixin(metaclass=serializers.SerializerMetaclass):
"""Mixin class which allows downloading an 'image' from a remote URL.
Adds the optional, write-only `remote_image` field to the serializer
"""
remote_image = serializers.URLField(
required=False,
allow_blank=False,
write_only=True,
label=_("URL"),
help_text=_("URL of remote image file"),
)
def validate_remote_image(self, url):
"""Perform custom validation for the remote image URL.
- Attempt to download the image and store it against this object instance
- Catches and re-throws any errors
"""
if not url:
return
if not InvenTreeSetting.get_setting('INVENTREE_DOWNLOAD_FROM_URL'):
raise ValidationError(_("Downloading images from remote URL is not enabled"))
try:
self.remote_image_file = download_image_from_url(url)
except Exception as exc:
self.remote_image_file = None
raise ValidationError(str(exc))
return url

View File

@ -13,6 +13,7 @@ from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.test import TestCase, override_settings
import requests
from djmoney.contrib.exchange.exceptions import MissingRate
from djmoney.contrib.exchange.models import Rate, convert_money
from djmoney.money import Money
@ -251,6 +252,45 @@ class TestHelpers(TestCase):
logo = helpers.getLogoImage(as_file=True)
self.assertEqual(logo, f'file://{settings.STATIC_ROOT}/img/inventree.png')
def test_download_image(self):
"""Test function for downloading image from remote URL"""
# Run check with a sequency of bad URLs
for url in [
"blog",
"htp://test.com/?",
"google",
"\\invalid-url"
]:
with self.assertRaises(django_exceptions.ValidationError):
helpers.download_image_from_url(url)
# Attempt to download an image which throws a 404
with self.assertRaises(requests.exceptions.HTTPError):
helpers.download_image_from_url("https://httpstat.us/404")
# Attempt to download, but timeout
with self.assertRaises(requests.exceptions.Timeout):
helpers.download_image_from_url("https://httpstat.us/200?sleep=5000")
# Attempt to download, but not a valid image
with self.assertRaises(TypeError):
helpers.download_image_from_url("https://httpstat.us/200")
large_img = "https://github.com/inventree/InvenTree/raw/master/InvenTree/InvenTree/static/img/paper_splash_large.jpg"
InvenTreeSetting.set_setting('INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE', 1, change_user=None)
# Attempt to download an image which is too large
with self.assertRaises(ValueError):
helpers.download_image_from_url(large_img)
# Increase allowable download size
InvenTreeSetting.set_setting('INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE', 5, change_user=None)
# Download a valid image (should not throw an error)
helpers.download_image_from_url(large_img)
class TestQuoteWrap(TestCase):
"""Tests for string wrapping."""

View File

@ -765,7 +765,6 @@ class NotificationsView(TemplateView):
# Custom 2FA removal form to allow custom redirect URL
class CustomTwoFactorRemove(TwoFactorRemove):
"""Specify custom URL redirect."""
success_url = reverse_lazy("settings")

View File

@ -24,7 +24,8 @@ from django.contrib.humanize.templatetags.humanize import naturaltime
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.core.exceptions import AppRegistryNotReady, ValidationError
from django.core.validators import MinValueValidator, URLValidator
from django.core.validators import (MaxValueValidator, MinValueValidator,
URLValidator)
from django.db import models, transaction
from django.db.utils import IntegrityError, OperationalError
from django.urls import reverse
@ -856,6 +857,18 @@ class InvenTreeSetting(BaseInvenTreeSetting):
'default': False,
},
'INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE': {
'name': _('Download Size Limit'),
'description': _('Maximum allowable download size for remote image'),
'units': 'MB',
'default': 1,
'validator': [
int,
MinValueValidator(1),
MaxValueValidator(25),
]
},
'INVENTREE_REQUIRE_CONFIRM': {
'name': _('Require confirm'),
'description': _('Require explicit user confirmation for certain action.'),

View File

@ -1,48 +0,0 @@
"""Django Forms for interacting with Company app."""
import django.forms
from django.utils.translation import gettext_lazy as _
from InvenTree.fields import RoundingDecimalFormField
from InvenTree.forms import HelperForm
from .models import Company, SupplierPriceBreak
class CompanyImageDownloadForm(HelperForm):
"""Form for downloading an image from a URL."""
url = django.forms.URLField(
label=_('URL'),
help_text=_('Image URL'),
required=True
)
class Meta:
"""Metaclass options."""
model = Company
fields = [
'url',
]
class EditPriceBreakForm(HelperForm):
"""Form for creating / editing a supplier price break."""
quantity = RoundingDecimalFormField(
max_digits=10,
decimal_places=5,
label=_('Quantity'),
help_text=_('Price break quantity'),
)
class Meta:
"""Metaclass options."""
model = SupplierPriceBreak
fields = [
'part',
'quantity',
'price',
]

View File

@ -1,5 +1,8 @@
"""JSON serializers for Company app."""
import io
from django.core.files.base import ContentFile
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
@ -11,7 +14,7 @@ from InvenTree.serializers import (InvenTreeAttachmentSerializer,
InvenTreeDecimalField,
InvenTreeImageSerializerField,
InvenTreeModelSerializer,
InvenTreeMoneySerializer)
InvenTreeMoneySerializer, RemoteImageMixin)
from part.serializers import PartBriefSerializer
from .models import (Company, ManufacturerPart, ManufacturerPartAttachment,
@ -39,7 +42,7 @@ class CompanyBriefSerializer(InvenTreeModelSerializer):
]
class CompanySerializer(InvenTreeModelSerializer):
class CompanySerializer(RemoteImageMixin, InvenTreeModelSerializer):
"""Serializer for Company object (full detail)"""
@staticmethod
@ -95,8 +98,33 @@ class CompanySerializer(InvenTreeModelSerializer):
'notes',
'parts_supplied',
'parts_manufactured',
'remote_image',
]
def save(self):
"""Save the Company instance"""
super().save()
company = self.instance
# Check if an image was downloaded from a remote URL
remote_img = getattr(self, 'remote_image_file', None)
if remote_img and company:
fmt = remote_img.format or 'PNG'
buffer = io.BytesIO()
remote_img.save(buffer, format=fmt)
# Construct a simplified name for the image
filename = f"company_{company.pk}_image.{fmt.lower()}"
company.image.save(
filename,
ContentFile(buffer.getvalue()),
)
return self.instance
class ManufacturerPartSerializer(InvenTreeModelSerializer):
"""Serializer for ManufacturerPart object."""

View File

@ -216,12 +216,19 @@
if (global_settings.INVENTREE_DOWNLOAD_FROM_URL) {
$('#company-image-url').click(function() {
launchModalForm(
'{% url "company-image-download" company.id %}',
constructForm(
'{% url "api-company-detail" company.pk %}',
{
reload: true,
method: 'PATCH',
title: '{% trans "Download Image" %}',
fields: {
remote_image: {},
},
onSuccess: function(data) {
reloadImage(data);
}
}
)
);
});
}

View File

@ -4,18 +4,12 @@ from django.urls import include, re_path
from . import views
company_detail_urls = [
re_path(r'^thumb-download/', views.CompanyImageDownloadFromURL.as_view(), name='company-image-download'),
# Any other URL
re_path(r'^.*$', views.CompanyDetail.as_view(), name='company-detail'),
]
company_urls = [
re_path(r'^(?P<pk>\d+)/', include(company_detail_urls)),
# Detail URLs for a specific Company instance
re_path(r'^(?P<pk>\d+)/', include([
re_path(r'^.*$', views.CompanyDetail.as_view(), name='company-detail'),
])),
re_path(r'suppliers/', views.CompanyIndex.as_view(), name='supplier-index'),
re_path(r'manufacturers/', views.CompanyIndex.as_view(), name='manufacturer-index'),

View File

@ -1,19 +1,12 @@
"""Django views for interacting with Company app."""
import io
from django.core.files.base import ContentFile
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django.views.generic import DetailView, ListView
import requests
from PIL import Image
from InvenTree.views import AjaxUpdateView, InvenTreeRoleMixin
from InvenTree.views import InvenTreeRoleMixin
from plugin.views import InvenTreePluginViewMixin
from .forms import CompanyImageDownloadForm
from .models import Company, ManufacturerPart, SupplierPart
@ -103,78 +96,6 @@ class CompanyDetail(InvenTreePluginViewMixin, DetailView):
permission_required = 'company.view_company'
class CompanyImageDownloadFromURL(AjaxUpdateView):
"""View for downloading an image from a provided URL."""
model = Company
ajax_template_name = 'image_download.html'
form_class = CompanyImageDownloadForm
ajax_form_title = _('Download Image')
def validate(self, company, form):
"""Validate that the image data are correct."""
# First ensure that the normal validation routines pass
if not form.is_valid():
return
# We can now extract a valid URL from the form data
url = form.cleaned_data.get('url', None)
# Download the file
response = requests.get(url, stream=True)
# Look at response header, reject if too large
content_length = response.headers.get('Content-Length', '0')
try:
content_length = int(content_length)
except (ValueError):
# If we cannot extract meaningful length, just assume it's "small enough"
content_length = 0
# TODO: Factor this out into a configurable setting
MAX_IMG_LENGTH = 10 * 1024 * 1024
if content_length > MAX_IMG_LENGTH:
form.add_error('url', _('Image size exceeds maximum allowable size for download'))
return
self.response = response
# Check for valid response code
if response.status_code != 200:
form.add_error('url', _('Invalid response: {code}').format(code=response.status_code))
return
response.raw.decode_content = True
try:
self.image = Image.open(response.raw).convert()
self.image.verify()
except Exception:
form.add_error('url', _("Supplied URL is not a valid image file"))
return
def save(self, company, form, **kwargs):
"""Save the downloaded image to the company."""
fmt = self.image.format
if not fmt:
fmt = 'PNG'
buffer = io.BytesIO()
self.image.save(buffer, format=fmt)
# Construct a simplified name for the image
filename = f"company_{company.pk}_image.{fmt.lower()}"
company.image.save(
filename,
ContentFile(buffer.getvalue()),
)
class ManufacturerPartDetail(InvenTreePluginViewMixin, DetailView):
"""Detail view for ManufacturerPart."""
model = ManufacturerPart

View File

@ -4,28 +4,9 @@ from django import forms
from django.utils.translation import gettext_lazy as _
from common.forms import MatchItemForm
from InvenTree.fields import RoundingDecimalFormField
from InvenTree.forms import HelperForm
from InvenTree.helpers import clean_decimal
from .models import Part, PartInternalPriceBreak, PartSellPriceBreak
class PartImageDownloadForm(HelperForm):
"""Form for downloading an image from a URL."""
url = forms.URLField(
label=_('URL'),
help_text=_('Image URL'),
required=True,
)
class Meta:
"""Metaclass defines fields for this form"""
model = Part
fields = [
'url',
]
from .models import Part
class BomMatchItemForm(MatchItemForm):
@ -66,33 +47,3 @@ class PartPriceForm(forms.Form):
fields = [
'quantity',
]
class EditPartSalePriceBreakForm(HelperForm):
"""Form for creating / editing a sale price for a part."""
quantity = RoundingDecimalFormField(max_digits=10, decimal_places=5, label=_('Quantity'))
class Meta:
"""Metaclass defines fields for this form"""
model = PartSellPriceBreak
fields = [
'part',
'quantity',
'price',
]
class EditPartInternalPriceBreakForm(HelperForm):
"""Form for creating / editing a internal price for a part."""
quantity = RoundingDecimalFormField(max_digits=10, decimal_places=5, label=_('Quantity'))
class Meta:
"""Metaclass defines fields for this form"""
model = PartInternalPriceBreak
fields = [
'part',
'quantity',
'price',
]

View File

@ -1,8 +1,10 @@
"""DRF data serializers for Part app."""
import imghdr
import io
from decimal import Decimal
from django.core.files.base import ContentFile
from django.db import models, transaction
from django.db.models import ExpressionWrapper, F, FloatField, Q
from django.db.models.functions import Coalesce
@ -22,7 +24,7 @@ from InvenTree.serializers import (DataFileExtractSerializer,
InvenTreeDecimalField,
InvenTreeImageSerializerField,
InvenTreeModelSerializer,
InvenTreeMoneySerializer)
InvenTreeMoneySerializer, RemoteImageMixin)
from InvenTree.status_codes import BuildStatus
from .models import (BomItem, BomItemSubstitute, Part, PartAttachment,
@ -273,7 +275,7 @@ class PartBriefSerializer(InvenTreeModelSerializer):
]
class PartSerializer(InvenTreeModelSerializer):
class PartSerializer(RemoteImageMixin, InvenTreeModelSerializer):
"""Serializer for complete detail information of a part.
Used when displaying all details of a single component.
@ -424,6 +426,7 @@ class PartSerializer(InvenTreeModelSerializer):
'parameters',
'pk',
'purchaseable',
'remote_image',
'revision',
'salable',
'starred',
@ -437,6 +440,31 @@ class PartSerializer(InvenTreeModelSerializer):
'virtual',
]
def save(self):
"""Save the Part instance"""
super().save()
part = self.instance
# Check if an image was downloaded from a remote URL
remote_img = getattr(self, 'remote_image_file', None)
if remote_img and part:
fmt = remote_img.format or 'PNG'
buffer = io.BytesIO()
remote_img.save(buffer, format=fmt)
# Construct a simplified name for the image
filename = f"part_{part.pk}_image.{fmt.lower()}"
part.image.save(
filename,
ContentFile(buffer.getvalue()),
)
return self.instance
class PartRelationSerializer(InvenTreeModelSerializer):
"""Serializer for a PartRelated model."""

View File

@ -511,11 +511,17 @@
{% if roles.part.change %}
if (global_settings.INVENTREE_DOWNLOAD_FROM_URL) {
$("#part-image-url").click(function() {
launchModalForm(
'{% url "part-image-download" part.id %}',
constructForm(
'{% url "api-part-detail" part.pk %}',
{
reload: true,
method: 'PATCH',
title: '{% trans "Download Image" %}',
fields: {
remote_image: {},
},
onSuccess: onSelectImage,
}
);
});

View File

@ -21,7 +21,6 @@ part_detail_urls = [
# Normal thumbnail with form
re_path(r'^thumb-select/?', views.PartImageSelect.as_view(), name='part-image-select'),
re_path(r'^thumb-download/', views.PartImageDownloadFromURL.as_view(), name='part-image-download'),
# Any other URLs go to the part detail page
re_path(r'^.*$', views.PartDetail.as_view(), name='part-detail'),

View File

@ -1,22 +1,18 @@
"""Django views for interacting with Part app."""
import io
import os
from decimal import Decimal
from django.conf import settings
from django.contrib import messages
from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.shortcuts import HttpResponseRedirect, get_object_or_404
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django.views.generic import DetailView, ListView
import requests
from djmoney.contrib.exchange.exceptions import MissingRate
from djmoney.contrib.exchange.models import convert_money
from PIL import Image
import common.settings as inventree_settings
from common.files import FileManager
@ -491,82 +487,6 @@ class PartQRCode(QRCodeView):
return None
class PartImageDownloadFromURL(AjaxUpdateView):
"""View for downloading an image from a provided URL."""
model = Part
ajax_template_name = 'image_download.html'
form_class = part_forms.PartImageDownloadForm
ajax_form_title = _('Download Image')
def validate(self, part, form):
"""Validate that the image data are correct.
- Try to download the image!
"""
# First ensure that the normal validation routines pass
if not form.is_valid():
return
# We can now extract a valid URL from the form data
url = form.cleaned_data.get('url', None)
# Download the file
response = requests.get(url, stream=True)
# Look at response header, reject if too large
content_length = response.headers.get('Content-Length', '0')
try:
content_length = int(content_length)
except (ValueError):
# If we cannot extract meaningful length, just assume it's "small enough"
content_length = 0
# TODO: Factor this out into a configurable setting
MAX_IMG_LENGTH = 10 * 1024 * 1024
if content_length > MAX_IMG_LENGTH:
form.add_error('url', _('Image size exceeds maximum allowable size for download'))
return
self.response = response
# Check for valid response code
if response.status_code != 200:
form.add_error('url', _('Invalid response: {code}').format(code=response.status_code))
return
response.raw.decode_content = True
try:
self.image = Image.open(response.raw).convert()
self.image.verify()
except Exception:
form.add_error('url', _("Supplied URL is not a valid image file"))
return
def save(self, part, form, **kwargs):
"""Save the downloaded image to the part."""
fmt = self.image.format
if not fmt:
fmt = 'PNG'
buffer = io.BytesIO()
self.image.save(buffer, format=fmt)
# Construct a simplified name for the image
filename = f"part_{part.pk}_image.{fmt.lower()}"
part.image.save(
filename,
ContentFile(buffer.getvalue()),
)
class PartImageSelect(AjaxUpdateView):
"""View for selecting Part image from existing images."""

View File

@ -13,13 +13,14 @@
<table class='table table-striped table-condensed'>
<tbody>
{% include "InvenTree/settings/setting.html" with key="INVENTREE_COMPANY_NAME" icon="fa-building" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_BASE_URL" icon="fa-globe" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_INSTANCE" icon="fa-info-circle" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_INSTANCE_TITLE" icon="fa-info-circle" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_RESTRICT_ABOUT" icon="fa-info-circle" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_BASE_URL" icon="fa-globe" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_COMPANY_NAME" icon="fa-building" %}
<tr><td colspan='5'></td></tr>
{% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_FROM_URL" icon="fa-cloud-download-alt" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE" icon="fa-server" %}
{% include "InvenTree/settings/setting.html" with key="INVENTREE_REQUIRE_CONFIRM" icon="fa-check" %}
</tbody>
</table>

View File

@ -1,16 +0,0 @@
{% extends "modal_form.html" %}
{% load inventree_extras %}
{% load i18n %}
{% block pre_form_content %}
<div class='alert alert-block alert-info'>
{% trans "Specify URL for downloading image" %}:
<ul>
<li>{% trans "Must be a valid image URL" %}</li>
<li>{% trans "Remote server must be accessible" %}</li>
<li>{% trans "Remote image must not exceed maximum allowable file size" %}</li>
</ul>
</div>
{% endblock %}