diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index d81e200dce..d94ab3b876 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,6 +41,7 @@ jobs: echo "git_commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_ENV echo "git_commit_date=$(git show -s --format=%ci)" >> $GITHUB_ENV - name: Build Docker Image + # Build the development docker image (using docker-compose.yml) run: | docker-compose build - name: Run Unit Tests @@ -51,6 +52,18 @@ jobs: docker-compose run inventree-dev-server invoke wait docker-compose run inventree-dev-server invoke test docker-compose down + - name: Check Data Directory + # The following file structure should have been created by the docker image + run: | + test -d data + test -d data/env + test -d data/pgdb + test -d data/media + test -d data/static + test -d data/plugins + test -f data/config.yaml + test -f data/plugins.txt + test -f data/secret_key.txt - name: Set up QEMU if: github.event_name != 'pull_request' uses: docker/setup-qemu-action@v1 @@ -58,6 +71,7 @@ jobs: if: github.event_name != 'pull_request' uses: docker/setup-buildx-action@v1 - name: Set up cosign + if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@48866aa521d8bf870604709cd43ec2f602d03ff2 - name: Login to Dockerhub if: github.event_name != 'pull_request' @@ -66,6 +80,7 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Extract Docker metadata + if: github.event_name != 'pull_request' id: meta uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a with: @@ -85,12 +100,13 @@ jobs: commit_hash=${{ env.git_commit_hash }} commit_date=${{ env.git_commit_date }} - name: Sign the published image + if: github.event_name != 'pull_request' env: COSIGN_EXPERIMENTAL: "true" run: cosign sign ${{ steps.meta.outputs.tags }}@${{ steps.build-and-push.outputs.digest }} - name: Push to Stable Branch uses: ad-m/github-push-action@master - if: env.stable_release == 'true' + if: env.stable_release == 'true' && github.event_name != 'pull_request' with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: stable diff --git a/.gitpod.yml b/.gitpod.yml index 4672cd7bd1..c6e891547b 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -7,11 +7,12 @@ tasks: export INVENTREE_STATIC_ROOT='/workspace/InvenTree/dev/static' export PIP_USER='no' + sudo apt install gettext python3 -m venv venv source venv/bin/activate pip install invoke mkdir dev - inv test-setup + inv setup-test gp sync-done start_server - name: Start server @@ -23,6 +24,7 @@ tasks: export INVENTREE_MEDIA_ROOT='/workspace/InvenTree/inventree-data/media' export INVENTREE_STATIC_ROOT='/workspace/InvenTree/dev/static' + source venv/bin/activate inv server ports: diff --git a/InvenTree/InvenTree/api_version.py b/InvenTree/InvenTree/api_version.py index 4a6cb3dba3..66dd1a0674 100644 --- a/InvenTree/InvenTree/api_version.py +++ b/InvenTree/InvenTree/api_version.py @@ -2,11 +2,15 @@ # InvenTree API version -INVENTREE_API_VERSION = 67 +INVENTREE_API_VERSION = 68 """ Increment this API version number whenever there is a significant change to the API that any clients need to know about +v68 -> 2022-07-27 : https://github.com/inventree/InvenTree/pull/3417 + - Allows SupplierPart list to be filtered by SKU value + - Allows SupplierPart list to be filtered by MPN value + v67 -> 2022-07-25 : https://github.com/inventree/InvenTree/pull/3395 - Adds a 'requirements' endpoint for Part instance - Provides information on outstanding order requirements for a given part diff --git a/InvenTree/InvenTree/config.py b/InvenTree/InvenTree/config.py index fe3e41a10f..e2fbe680cd 100644 --- a/InvenTree/InvenTree/config.py +++ b/InvenTree/InvenTree/config.py @@ -3,16 +3,17 @@ import logging import os import shutil +from pathlib import Path logger = logging.getLogger('inventree') -def get_base_dir(): +def get_base_dir() -> Path: """Returns the base (top-level) InvenTree directory.""" - return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + return Path(__file__).parent.parent.resolve() -def get_config_file(): +def get_config_file() -> Path: """Returns the path of the InvenTree configuration file. Note: It will be created it if does not already exist! @@ -22,16 +23,15 @@ def get_config_file(): cfg_filename = os.getenv('INVENTREE_CONFIG_FILE') if cfg_filename: - cfg_filename = cfg_filename.strip() - cfg_filename = os.path.abspath(cfg_filename) + cfg_filename = Path(cfg_filename.strip()).resolve() else: # Config file is *not* specified - use the default - cfg_filename = os.path.join(base_dir, 'config.yaml') + cfg_filename = base_dir.joinpath('config.yaml').resolve() - if not os.path.exists(cfg_filename): + if not cfg_filename.exists(): print("InvenTree configuration file 'config.yaml' not found - creating default file") - cfg_template = os.path.join(base_dir, "config_template.yaml") + cfg_template = base_dir.joinpath("config_template.yaml") shutil.copyfile(cfg_template, cfg_filename) print(f"Created config file {cfg_filename}") @@ -48,18 +48,18 @@ def get_plugin_file(): if not PLUGIN_FILE: # If not specified, look in the same directory as the configuration file + config_dir = get_config_file().parent + PLUGIN_FILE = config_dir.joinpath('plugins.txt') + else: + # Make sure we are using a modern Path object + PLUGIN_FILE = Path(PLUGIN_FILE) - config_dir = os.path.dirname(get_config_file()) - - PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') - - if not os.path.exists(PLUGIN_FILE): + if not PLUGIN_FILE.exists(): logger.warning("Plugin configuration file does not exist") logger.info(f"Creating plugin file at '{PLUGIN_FILE}'") # If opening the file fails (no write permission, for example), then this will throw an error - with open(PLUGIN_FILE, 'w') as plugin_file: - plugin_file.write("# InvenTree Plugins (uses PIP framework to install)\n\n") + PLUGIN_FILE.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n") return PLUGIN_FILE diff --git a/InvenTree/InvenTree/fields.py b/InvenTree/InvenTree/fields.py index 6c62aacb9d..35f9fb5eb4 100644 --- a/InvenTree/InvenTree/fields.py +++ b/InvenTree/InvenTree/fields.py @@ -12,12 +12,21 @@ from django.utils.translation import gettext_lazy as _ from djmoney.forms.fields import MoneyField from djmoney.models.fields import MoneyField as ModelMoneyField from djmoney.models.validators import MinMoneyValidator +from rest_framework.fields import URLField as RestURLField import InvenTree.helpers from .validators import allowable_url_schemes +class InvenTreeRestURLField(RestURLField): + """Custom field for DRF with custom scheme vaildators.""" + def __init__(self, **kwargs): + """Update schemes.""" + super().__init__(**kwargs) + self.validators[-1].schemes = allowable_url_schemes() + + class InvenTreeURLFormField(FormURLField): """Custom URL form field with custom scheme validators.""" @@ -27,7 +36,7 @@ class InvenTreeURLFormField(FormURLField): class InvenTreeURLField(models.URLField): """Custom URL field which has custom scheme validators.""" - default_validators = [validators.URLValidator(schemes=allowable_url_schemes())] + validators = [validators.URLValidator(schemes=allowable_url_schemes())] def formfield(self, **kwargs): """Return a Field instance for this field.""" diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 85d9a0e5b0..0d2f6d9ab7 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -7,6 +7,7 @@ import os import os.path import re from decimal import Decimal, InvalidOperation +from pathlib import Path from wsgiref.util import FileWrapper from django.conf import settings @@ -211,7 +212,7 @@ def getLogoImage(as_file=False, custom=True): else: if as_file: - path = os.path.join(settings.STATIC_ROOT, 'img/inventree.png') + path = settings.STATIC_ROOT.joinpath('img/inventree.png') return f"file://{path}" else: return getStaticUrl('img/inventree.png') @@ -687,20 +688,17 @@ def addUserPermissions(user, permissions): def getMigrationFileNames(app): """Return a list of all migration filenames for provided app.""" - local_dir = os.path.dirname(os.path.abspath(__file__)) - - migration_dir = os.path.join(local_dir, '..', app, 'migrations') - - files = os.listdir(migration_dir) + local_dir = Path(__file__).parent + files = local_dir.joinpath('..', app, 'migrations').iterdir() # Regex pattern for migration files - pattern = r"^[\d]+_.*\.py$" + regex = re.compile(r"^[\d]+_.*\.py$") migration_files = [] for f in files: - if re.match(pattern, f): - migration_files.append(f) + if regex.match(f.name): + migration_files.append(f.name) return migration_files diff --git a/InvenTree/InvenTree/models.py b/InvenTree/InvenTree/models.py index 889eea0176..e8226cc5fb 100644 --- a/InvenTree/InvenTree/models.py +++ b/InvenTree/InvenTree/models.py @@ -437,26 +437,12 @@ class InvenTreeAttachment(models.Model): if len(fn) == 0: raise ValidationError(_('Filename must not be empty')) - attachment_dir = os.path.join( - settings.MEDIA_ROOT, - self.getSubdir() - ) - - old_file = os.path.join( - settings.MEDIA_ROOT, - self.attachment.name - ) - - new_file = os.path.join( - settings.MEDIA_ROOT, - self.getSubdir(), - fn - ) - - new_file = os.path.abspath(new_file) + attachment_dir = settings.MEDIA_ROOT.joinpath(self.getSubdir()) + old_file = settings.MEDIA_ROOT.joinpath(self.attachment.name) + new_file = settings.MEDIA_ROOT.joinpath(self.getSubdir(), fn).resolve() # Check that there are no directory tricks going on... - if os.path.dirname(new_file) != attachment_dir: + if new_file.parent != attachment_dir: logger.error(f"Attempted to rename attachment outside valid directory: '{new_file}'") raise ValidationError(_("Invalid attachment directory")) @@ -473,11 +459,11 @@ class InvenTreeAttachment(models.Model): if len(fn.split('.')) < 2: raise ValidationError(_("Filename missing extension")) - if not os.path.exists(old_file): + if not old_file.exists(): logger.error(f"Trying to rename attachment '{old_file}' which does not exist") return - if os.path.exists(new_file): + if new_file.exists(): raise ValidationError(_("Attachment with this filename already exists")) try: diff --git a/InvenTree/InvenTree/serializers.py b/InvenTree/InvenTree/serializers.py index d890f9f478..385d2f4837 100644 --- a/InvenTree/InvenTree/serializers.py +++ b/InvenTree/InvenTree/serializers.py @@ -7,6 +7,7 @@ from decimal import Decimal from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError as DjangoValidationError +from django.db import models from django.utils.translation import gettext_lazy as _ import tablib @@ -20,6 +21,7 @@ from rest_framework.serializers import DecimalField from rest_framework.utils import model_meta from common.models import InvenTreeSetting +from InvenTree.fields import InvenTreeRestURLField from InvenTree.helpers import download_image_from_url @@ -64,6 +66,12 @@ class InvenTreeMoneySerializer(MoneyField): class InvenTreeModelSerializer(serializers.ModelSerializer): """Inherits the standard Django ModelSerializer class, but also ensures that the underlying model class data are checked on validation.""" + # Switch out URLField mapping + serializer_field_mapping = { + **serializers.ModelSerializer.serializer_field_mapping, + models.URLField: InvenTreeRestURLField, + } + def __init__(self, instance=None, data=empty, **kwargs): """Custom __init__ routine to ensure that *default* values (as specified in the ORM) are used by the DRF serializers, *if* the values are not provided by the user.""" # If instance is None, we are creating a new instance diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index eb18342bd1..6b28d1e6eb 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -15,6 +15,7 @@ import random import socket import string import sys +from pathlib import Path import django.conf.locale from django.core.files.storage import default_storage @@ -44,7 +45,7 @@ TESTING_ENV = False # New requirement for django 3.2+ DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +# Build paths inside the project like this: BASE_DIR.joinpath(...) BASE_DIR = get_base_dir() cfg_filename = get_config_file() @@ -53,7 +54,7 @@ with open(cfg_filename, 'r') as cfg: CONFIG = yaml.safe_load(cfg) # We will place any config files in the same directory as the config file -config_dir = os.path.dirname(cfg_filename) +config_dir = cfg_filename.parent # Default action is to run the system in Debug mode # SECURITY WARNING: don't run with debug turned on in production! @@ -123,19 +124,17 @@ else: key_file = os.getenv("INVENTREE_SECRET_KEY_FILE") if key_file: - key_file = os.path.abspath(key_file) # pragma: no cover + key_file = Path(key_file).resolve() # pragma: no cover else: # default secret key location - key_file = os.path.join(BASE_DIR, "secret_key.txt") - key_file = os.path.abspath(key_file) + key_file = BASE_DIR.joinpath("secret_key.txt").resolve() - if not os.path.exists(key_file): # pragma: no cover + if not key_file.exists(): # pragma: no cover logger.info(f"Generating random key file at '{key_file}'") # Create a random key file - with open(key_file, 'w') as f: - options = string.digits + string.ascii_letters + string.punctuation - key = ''.join([random.choice(options) for i in range(100)]) - f.write(key) + options = string.digits + string.ascii_letters + string.punctuation + key = ''.join([random.choice(options) for i in range(100)]) + key_file.write_text(key) logger.info(f"Loading SECRET_KEY from '{key_file}'") @@ -146,28 +145,34 @@ else: sys.exit(-1) # The filesystem location for served static files -STATIC_ROOT = os.path.abspath( +STATIC_ROOT = Path( get_setting( 'INVENTREE_STATIC_ROOT', CONFIG.get('static_root', None) ) -) +).resolve() if STATIC_ROOT is None: # pragma: no cover print("ERROR: INVENTREE_STATIC_ROOT directory not defined") sys.exit(1) +else: + # Ensure the root really is availalble + STATIC_ROOT.mkdir(parents=True, exist_ok=True) # The filesystem location for served static files -MEDIA_ROOT = os.path.abspath( +MEDIA_ROOT = Path( get_setting( 'INVENTREE_MEDIA_ROOT', CONFIG.get('media_root', None) ) -) +).resolve() if MEDIA_ROOT is None: # pragma: no cover print("ERROR: INVENTREE_MEDIA_ROOT directory is not defined") sys.exit(1) +else: + # Ensure the root really is availalble + MEDIA_ROOT.mkdir(parents=True, exist_ok=True) # List of allowed hosts (default = allow all) ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*']) @@ -193,17 +198,17 @@ STATICFILES_DIRS = [] # Translated Template settings STATICFILES_I18_PREFIX = 'i18n' -STATICFILES_I18_SRC = os.path.join(BASE_DIR, 'templates', 'js', 'translated') -STATICFILES_I18_TRG = os.path.join(BASE_DIR, 'InvenTree', 'static_i18n') +STATICFILES_I18_SRC = BASE_DIR.joinpath('templates', 'js', 'translated') +STATICFILES_I18_TRG = BASE_DIR.joinpath('InvenTree', 'static_i18n') STATICFILES_DIRS.append(STATICFILES_I18_TRG) -STATICFILES_I18_TRG = os.path.join(STATICFILES_I18_TRG, STATICFILES_I18_PREFIX) +STATICFILES_I18_TRG = STATICFILES_I18_TRG.joinpath(STATICFILES_I18_PREFIX) STATFILES_I18_PROCESSORS = [ 'InvenTree.context.status_codes', ] # Color Themes Directory -STATIC_COLOR_THEMES_DIR = os.path.join(STATIC_ROOT, 'css', 'color-themes') +STATIC_COLOR_THEMES_DIR = STATIC_ROOT.joinpath('css', 'color-themes') # Web URL endpoint for served media files MEDIA_URL = '/media/' @@ -339,10 +344,10 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [ - os.path.join(BASE_DIR, 'templates'), + BASE_DIR.joinpath('templates'), # Allow templates in the reporting directory to be accessed - os.path.join(MEDIA_ROOT, 'report'), - os.path.join(MEDIA_ROOT, 'label'), + MEDIA_ROOT.joinpath('report'), + MEDIA_ROOT.joinpath('label'), ], 'OPTIONS': { 'context_processors': [ @@ -809,7 +814,7 @@ EMAIL_USE_SSL = get_setting( EMAIL_TIMEOUT = 60 LOCALE_PATHS = ( - os.path.join(BASE_DIR, 'locale/'), + BASE_DIR.joinpath('locale/'), ) TIME_ZONE = get_setting( @@ -935,17 +940,6 @@ PLUGINS_ENABLED = _is_true(get_setting( PLUGIN_FILE = get_plugin_file() -# Plugin Directories (local plugins will be loaded from these directories) -PLUGIN_DIRS = ['plugin.builtin', ] - -if not TESTING: - # load local deploy directory in prod - PLUGIN_DIRS.append('plugins') # pragma: no cover - -if DEBUG or TESTING: - # load samples in debug mode - PLUGIN_DIRS.append('plugin.samples') - # Plugin test settings PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING) # are plugins beeing tested? PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False) # load plugins from setup hooks in testing? diff --git a/InvenTree/InvenTree/test_views.py b/InvenTree/InvenTree/test_views.py index 43e9e7782d..79c029941d 100644 --- a/InvenTree/InvenTree/test_views.py +++ b/InvenTree/InvenTree/test_views.py @@ -2,6 +2,7 @@ import os +from django.contrib.auth import get_user_model from django.urls import reverse from InvenTree.helpers import InvenTreeTestCase @@ -41,3 +42,80 @@ class ViewTests(InvenTreeTestCase): self.assertIn("
", content) # TODO: In future, run the javascript and ensure that the panels get created! + + def test_settings_page(self): + """Test that the 'settings' page loads correctly""" + + # Settings page loads + url = reverse('settings') + + # Attempt without login + self.client.logout() + response = self.client.get(url) + self.assertEqual(response.status_code, 302) + + # Login with default client + self.client.login(username=self.username, password=self.password) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + content = response.content.decode() + + user_panels = [ + 'account', + 'user-display', + 'user-home', + 'user-reports', + ] + + staff_panels = [ + 'server', + 'login', + 'barcodes', + 'currencies', + 'parts', + 'stock', + ] + + plugin_panels = [ + 'plugin', + ] + + # Default user has staff access, so all panels will be present + for panel in user_panels + staff_panels + plugin_panels: + self.assertIn(f"select-{panel}", content) + self.assertIn(f"panel-{panel}", content) + + # Now create a user who does not have staff access + pleb_user = get_user_model().objects.create_user( + username='pleb', + password='notstaff', + ) + + pleb_user.groups.add(self.group) + pleb_user.is_superuser = False + pleb_user.is_staff = False + pleb_user.save() + + self.client.logout() + + result = self.client.login( + username='pleb', + password='notstaff', + ) + + self.assertTrue(result) + + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + content = response.content.decode() + + # Normal user still has access to user-specific panels + for panel in user_panels: + self.assertIn(f"select-{panel}", content) + self.assertIn(f"panel-{panel}", content) + + # Normal user does NOT have access to global or plugin settings + for panel in staff_panels + plugin_panels: + self.assertNotIn(f"select-{panel}", content) + self.assertNotIn(f"panel-{panel}", content) diff --git a/InvenTree/InvenTree/tests.py b/InvenTree/InvenTree/tests.py index d4291268fa..b6d46bea80 100644 --- a/InvenTree/InvenTree/tests.py +++ b/InvenTree/InvenTree/tests.py @@ -745,11 +745,11 @@ class TestSettings(helpers.InvenTreeTestCase): 'inventree/data/config.yaml', ] - self.assertTrue(any([opt in config.get_config_file().lower() for opt in valid])) + self.assertTrue(any([opt in str(config.get_config_file()).lower() for opt in valid])) # with env set with self.in_env_context({'INVENTREE_CONFIG_FILE': 'my_special_conf.yaml'}): - self.assertIn('inventree/my_special_conf.yaml', config.get_config_file().lower()) + self.assertIn('inventree/my_special_conf.yaml', str(config.get_config_file()).lower()) def test_helpers_plugin_file(self): """Test get_plugin_file.""" @@ -760,11 +760,11 @@ class TestSettings(helpers.InvenTreeTestCase): 'inventree/data/plugins.txt', ] - self.assertTrue(any([opt in config.get_plugin_file().lower() for opt in valid])) + self.assertTrue(any([opt in str(config.get_plugin_file()).lower() for opt in valid])) # with env set with self.in_env_context({'INVENTREE_PLUGIN_FILE': 'my_special_plugins.txt'}): - self.assertIn('my_special_plugins.txt', config.get_plugin_file()) + self.assertIn('my_special_plugins.txt', str(config.get_plugin_file())) def test_helpers_setting(self): """Test get_setting.""" diff --git a/InvenTree/InvenTree/views.py b/InvenTree/InvenTree/views.py index 95f730fc7a..6c2b0e8779 100644 --- a/InvenTree/InvenTree/views.py +++ b/InvenTree/InvenTree/views.py @@ -5,7 +5,6 @@ as JSON objects and passing them to modal forms (using jQuery / bootstrap). """ import json -import os from django.conf import settings from django.contrib.auth import password_validation @@ -638,7 +637,8 @@ class SettingsView(TemplateView): ctx["rates_updated"] = None # load locale stats - STAT_FILE = os.path.abspath(os.path.join(settings.BASE_DIR, 'InvenTree/locale_stats.json')) + STAT_FILE = settings.BASE_DIR.joinpath('InvenTree/locale_stats.json').absolute() + try: ctx["locale_stats"] = json.load(open(STAT_FILE, 'r')) except Exception: diff --git a/InvenTree/common/views.py b/InvenTree/common/views.py index b356c5cb94..62e8011141 100644 --- a/InvenTree/common/views.py +++ b/InvenTree/common/views.py @@ -1,7 +1,5 @@ """Django views for interacting with common models.""" -import os - from django.conf import settings from django.core.files.storage import FileSystemStorage from django.utils.translation import gettext_lazy as _ @@ -37,9 +35,9 @@ class MultiStepFormView(SessionWizardView): def process_media_folder(self): """Process media folder.""" if self.media_folder: - media_folder_abs = os.path.join(settings.MEDIA_ROOT, self.media_folder) - if not os.path.exists(media_folder_abs): - os.mkdir(media_folder_abs) + media_folder_abs = settings.MEDIA_ROOT.joinpath(self.media_folder) + if not media_folder_abs.exists(): + media_folder_abs.mkdir(parents=True, exist_ok=True) self.file_storage = FileSystemStorage(location=media_folder_abs) def get_template_names(self): diff --git a/InvenTree/company/api.py b/InvenTree/company/api.py index 734dd08b5e..3307c9a619 100644 --- a/InvenTree/company/api.py +++ b/InvenTree/company/api.py @@ -254,6 +254,31 @@ class ManufacturerPartParameterDetail(RetrieveUpdateDestroyAPI): serializer_class = ManufacturerPartParameterSerializer +class SupplierPartFilter(rest_filters.FilterSet): + """API filters for the SupplierPartList endpoint""" + + class Meta: + """Metaclass option""" + + model = SupplierPart + fields = [ + 'supplier', + 'part', + 'manufacturer_part', + 'SKU', + ] + + # Filter by 'active' status of linked part + active = rest_filters.BooleanFilter(field_name='part__active') + + # Filter by the 'MPN' of linked manufacturer part + MPN = rest_filters.CharFilter( + label='Manufacturer Part Number', + field_name='manufacturer_part__MPN', + lookup_expr='iexact' + ) + + class SupplierPartList(ListCreateDestroyAPIView): """API endpoint for list view of SupplierPart object. @@ -262,6 +287,7 @@ class SupplierPartList(ListCreateDestroyAPIView): """ queryset = SupplierPart.objects.all() + filterset_class = SupplierPartFilter def get_queryset(self, *args, **kwargs): """Return annotated queryest object for the SupplierPart list""" @@ -282,37 +308,12 @@ class SupplierPartList(ListCreateDestroyAPIView): if manufacturer is not None: queryset = queryset.filter(manufacturer_part__manufacturer=manufacturer) - # Filter by supplier - supplier = params.get('supplier', None) - - if supplier is not None: - queryset = queryset.filter(supplier=supplier) - # Filter by EITHER manufacturer or supplier company = params.get('company', None) if company is not None: queryset = queryset.filter(Q(manufacturer_part__manufacturer=company) | Q(supplier=company)) - # Filter by parent part? - part = params.get('part', None) - - if part is not None: - queryset = queryset.filter(part=part) - - # Filter by manufacturer part? - manufacturer_part = params.get('manufacturer_part', None) - - if manufacturer_part is not None: - queryset = queryset.filter(manufacturer_part=manufacturer_part) - - # Filter by 'active' status of the part? - active = params.get('active', None) - - if active is not None: - active = str2bool(active) - queryset = queryset.filter(part__active=active) - return queryset def get_serializer(self, *args, **kwargs): diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index ab37e2f275..aacf020924 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -5,6 +5,7 @@ import logging import os import shutil import warnings +from pathlib import Path from django.apps import AppConfig from django.conf import settings @@ -40,262 +41,137 @@ class LabelConfig(AppConfig): """Create all default templates.""" # Test if models are ready try: - from .models import StockLocationLabel + from .models import PartLabel, StockItemLabel, StockLocationLabel assert bool(StockLocationLabel is not None) except AppRegistryNotReady: # pragma: no cover # Database might not yet be ready warnings.warn('Database was not ready for creating labels') return - self.create_stock_item_labels() - self.create_stock_location_labels() - self.create_part_labels() - - def create_stock_item_labels(self): - """Create database entries for the default StockItemLabel templates, if they do not already exist.""" - from .models import StockItemLabel - - src_dir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'templates', - 'label', + # Create the categories + self.create_labels_category( + StockItemLabel, 'stockitem', + [ + { + 'file': 'qr.html', + 'name': 'QR Code', + 'description': 'Simple QR code label', + 'width': 24, + 'height': 24, + }, + ], + ) + self.create_labels_category( + StockLocationLabel, + 'stocklocation', + [ + { + 'file': 'qr.html', + 'name': 'QR Code', + 'description': 'Simple QR code label', + 'width': 24, + 'height': 24, + }, + { + 'file': 'qr_and_text.html', + 'name': 'QR and text', + 'description': 'Label with QR code and name of location', + 'width': 50, + 'height': 24, + } + ] + ) + self.create_labels_category( + PartLabel, + 'part', + [ + { + 'file': 'part_label.html', + 'name': 'Part Label', + 'description': 'Simple part label', + 'width': 70, + 'height': 24, + }, + { + 'file': 'part_label_code128.html', + 'name': 'Barcode Part Label', + 'description': 'Simple part label with Code128 barcode', + 'width': 70, + 'height': 24, + }, + ] ) - dst_dir = os.path.join( - settings.MEDIA_ROOT, - 'label', - 'inventree', - 'stockitem', - ) - - if not os.path.exists(dst_dir): - logger.info(f"Creating required directory: '{dst_dir}'") - os.makedirs(dst_dir, exist_ok=True) - - labels = [ - { - 'file': 'qr.html', - 'name': 'QR Code', - 'description': 'Simple QR code label', - 'width': 24, - 'height': 24, - }, - ] - - for label in labels: - - filename = os.path.join( - 'label', - 'inventree', - 'stockitem', - label['file'], - ) - - # Check if the file exists in the media directory - src_file = os.path.join(src_dir, label['file']) - dst_file = os.path.join(settings.MEDIA_ROOT, filename) - - to_copy = False - - if os.path.exists(dst_file): - # File already exists - let's see if it is the "same", - # or if we need to overwrite it with a newer copy! - - if hashFile(dst_file) != hashFile(src_file): # pragma: no cover - logger.info(f"Hash differs for '{filename}'") - to_copy = True - - else: - logger.info(f"Label template '{filename}' is not present") - to_copy = True - - if to_copy: - logger.info(f"Copying label template '{dst_file}'") - shutil.copyfile(src_file, dst_file) - - # Check if a label matching the template already exists - if StockItemLabel.objects.filter(label=filename).exists(): - continue # pragma: no cover - - logger.info(f"Creating entry for StockItemLabel '{label['name']}'") - - StockItemLabel.objects.create( - name=label['name'], - description=label['description'], - label=filename, - filters='', - enabled=True, - width=label['width'], - height=label['height'], - ) - - def create_stock_location_labels(self): - """Create database entries for the default StockItemLocation templates, if they do not already exist.""" - from .models import StockLocationLabel - - src_dir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), + def create_labels_category(self, model, ref_name, labels): + """Create folder and database entries for the default templates, if they do not already exist.""" + # Create root dir for templates + src_dir = Path(__file__).parent.joinpath( 'templates', 'label', - 'stocklocation', + ref_name, ) - dst_dir = os.path.join( - settings.MEDIA_ROOT, + dst_dir = settings.MEDIA_ROOT.joinpath( 'label', 'inventree', - 'stocklocation', + ref_name, ) - if not os.path.exists(dst_dir): + if not dst_dir.exists(): logger.info(f"Creating required directory: '{dst_dir}'") - os.makedirs(dst_dir, exist_ok=True) - - labels = [ - { - 'file': 'qr.html', - 'name': 'QR Code', - 'description': 'Simple QR code label', - 'width': 24, - 'height': 24, - }, - { - 'file': 'qr_and_text.html', - 'name': 'QR and text', - 'description': 'Label with QR code and name of location', - 'width': 50, - 'height': 24, - } - ] + dst_dir.mkdir(parents=True, exist_ok=True) + # Create lables for label in labels: + self.create_template_label(model, src_dir, ref_name, label) - filename = os.path.join( - 'label', - 'inventree', - 'stocklocation', - label['file'], - ) - - # Check if the file exists in the media directory - src_file = os.path.join(src_dir, label['file']) - dst_file = os.path.join(settings.MEDIA_ROOT, filename) - - to_copy = False - - if os.path.exists(dst_file): - # File already exists - let's see if it is the "same", - # or if we need to overwrite it with a newer copy! - - if hashFile(dst_file) != hashFile(src_file): # pragma: no cover - logger.info(f"Hash differs for '{filename}'") - to_copy = True - - else: - logger.info(f"Label template '{filename}' is not present") - to_copy = True - - if to_copy: - logger.info(f"Copying label template '{dst_file}'") - shutil.copyfile(src_file, dst_file) - - # Check if a label matching the template already exists - if StockLocationLabel.objects.filter(label=filename).exists(): - continue # pragma: no cover - - logger.info(f"Creating entry for StockLocationLabel '{label['name']}'") - - StockLocationLabel.objects.create( - name=label['name'], - description=label['description'], - label=filename, - filters='', - enabled=True, - width=label['width'], - height=label['height'], - ) - - def create_part_labels(self): - """Create database entries for the default PartLabel templates, if they do not already exist.""" - from .models import PartLabel - - src_dir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'templates', - 'label', - 'part', - ) - - dst_dir = os.path.join( - settings.MEDIA_ROOT, + def create_template_label(self, model, src_dir, ref_name, label): + """Ensure a label template is in place.""" + filename = os.path.join( 'label', 'inventree', - 'part', + ref_name, + label['file'] ) - if not os.path.exists(dst_dir): - logger.info(f"Creating required directory: '{dst_dir}'") - os.makedirs(dst_dir, exist_ok=True) + src_file = src_dir.joinpath(label['file']) + dst_file = settings.MEDIA_ROOT.joinpath(filename) - labels = [ - { - 'file': 'part_label.html', - 'name': 'Part Label', - 'description': 'Simple part label', - 'width': 70, - 'height': 24, - }, - { - 'file': 'part_label_code128.html', - 'name': 'Barcode Part Label', - 'description': 'Simple part label with Code128 barcode', - 'width': 70, - 'height': 24, - }, - ] + to_copy = False - for label in labels: + if dst_file.exists(): + # File already exists - let's see if it is the "same" - filename = os.path.join( - 'label', - 'inventree', - 'part', - label['file'] - ) - - src_file = os.path.join(src_dir, label['file']) - dst_file = os.path.join(settings.MEDIA_ROOT, filename) - - to_copy = False - - if os.path.exists(dst_file): - # File already exists - let's see if it is the "same" - - if hashFile(dst_file) != hashFile(src_file): # pragma: no cover - logger.info(f"Hash differs for '{filename}'") - to_copy = True - - else: - logger.info(f"Label template '{filename}' is not present") + if hashFile(dst_file) != hashFile(src_file): # pragma: no cover + logger.info(f"Hash differs for '{filename}'") to_copy = True - if to_copy: - logger.info(f"Copying label template '{dst_file}'") - shutil.copyfile(src_file, dst_file) + else: + logger.info(f"Label template '{filename}' is not present") + to_copy = True - # Check if a label matching the template already exists - if PartLabel.objects.filter(label=filename).exists(): - continue # pragma: no cover + if to_copy: + logger.info(f"Copying label template '{dst_file}'") + # Ensure destionation dir exists + dst_file.parent.mkdir(parents=True, exist_ok=True) - logger.info(f"Creating entry for PartLabel '{label['name']}'") + # Copy file + shutil.copyfile(src_file, dst_file) - PartLabel.objects.create( - name=label['name'], - description=label['description'], - label=filename, - filters='', - enabled=True, - width=label['width'], - height=label['height'], - ) + # Check if a label matching the template already exists + if model.objects.filter(label=filename).exists(): + return # pragma: no cover + + logger.info(f"Creating entry for {model} '{label['name']}'") + + model.objects.create( + name=label['name'], + description=label['description'], + label=filename, + filters='', + enabled=True, + width=label['width'], + height=label['height'], + ) + return diff --git a/InvenTree/label/models.py b/InvenTree/label/models.py index 03dcd7468b..13b5ac331e 100644 --- a/InvenTree/label/models.py +++ b/InvenTree/label/models.py @@ -155,7 +155,7 @@ class LabelTemplate(models.Model): template = template.replace('/', os.path.sep) template = template.replace('\\', os.path.sep) - template = os.path.join(settings.MEDIA_ROOT, template) + template = settings.MEDIA_ROOT.joinpath(template) return template diff --git a/InvenTree/label/tests.py b/InvenTree/label/tests.py index 7de7fd6c80..4a894ac791 100644 --- a/InvenTree/label/tests.py +++ b/InvenTree/label/tests.py @@ -1,7 +1,5 @@ """Tests for labels""" -import os - from django.apps import apps from django.conf import settings from django.core.exceptions import ValidationError @@ -42,27 +40,17 @@ class LabelTest(InvenTreeAPITestCase): def test_default_files(self): """Test that label files exist in the MEDIA directory.""" - item_dir = os.path.join( - settings.MEDIA_ROOT, - 'label', - 'inventree', - 'stockitem', - ) + def test_subdir(ref_name): + item_dir = settings.MEDIA_ROOT.joinpath( + 'label', + 'inventree', + ref_name, + ) + self.assertTrue(len([item_dir.iterdir()]) > 0) - files = os.listdir(item_dir) - - self.assertTrue(len(files) > 0) - - loc_dir = os.path.join( - settings.MEDIA_ROOT, - 'label', - 'inventree', - 'stocklocation', - ) - - files = os.listdir(loc_dir) - - self.assertTrue(len(files) > 0) + test_subdir('stockitem') + test_subdir('stocklocation') + test_subdir('part') def test_filters(self): """Test the label filters.""" diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index abab3a4e9e..5d2e0605c7 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -568,7 +568,7 @@ class PartScheduling(RetrieveAPI): class PartRequirements(RetrieveAPI): - """API endpoint detailing 'requirements' information for aa particular part. + """API endpoint detailing 'requirements' information for a particular part. This endpoint returns information on upcoming requirements for: diff --git a/InvenTree/part/views.py b/InvenTree/part/views.py index 91569084e0..876d54e1c3 100644 --- a/InvenTree/part/views.py +++ b/InvenTree/part/views.py @@ -510,7 +510,7 @@ class PartImageSelect(AjaxUpdateView): data = {} if img: - img_path = os.path.join(settings.MEDIA_ROOT, 'part_images', img) + img_path = settings.MEDIA_ROOT.joinpath('part_images', img) # Ensure that the image already exists if os.path.exists(img_path): diff --git a/InvenTree/plugin/helpers.py b/InvenTree/plugin/helpers.py index 53326ab286..89edb1a571 100644 --- a/InvenTree/plugin/helpers.py +++ b/InvenTree/plugin/helpers.py @@ -2,7 +2,6 @@ import inspect import logging -import os import pathlib import pkgutil import subprocess @@ -103,10 +102,10 @@ def get_git_log(path): output = None if registry.git_is_modern: - path = path.replace(os.path.dirname(settings.BASE_DIR), '')[1:] + path = path.replace(str(settings.BASE_DIR.parent), '')[1:] command = ['git', 'log', '-n', '1', "--pretty=format:'%H%n%aN%n%aE%n%aI%n%f%n%G?%n%GK'", '--follow', '--', path] try: - output = str(subprocess.check_output(command, cwd=os.path.dirname(settings.BASE_DIR)), 'utf-8')[1:-1] + output = str(subprocess.check_output(command, cwd=settings.BASE_DIR.parent), 'utf-8')[1:-1] if output: output = output.split('\n') except subprocess.CalledProcessError: # pragma: no cover @@ -125,7 +124,7 @@ def check_git_version(): """Returns if the current git version supports modern features.""" # get version string try: - output = str(subprocess.check_output(['git', '--version'], cwd=os.path.dirname(settings.BASE_DIR)), 'utf-8') + output = str(subprocess.check_output(['git', '--version'], cwd=settings.BASE_DIR.parent), 'utf-8') except subprocess.CalledProcessError: # pragma: no cover return False except FileNotFoundError: # pragma: no cover @@ -172,10 +171,16 @@ class GitStatus: # region plugin finders -def get_modules(pkg): +def get_modules(pkg, path=None): """Get all modules in a package.""" context = {} - for loader, name, _ in pkgutil.walk_packages(pkg.__path__): + + if path is None: + path = pkg.__path__ + elif type(path) is not list: + path = [path] + + for loader, name, _ in pkgutil.walk_packages(path): try: module = loader.find_module(name).load_module(name) pkg_names = getattr(module, '__all__', None) @@ -199,7 +204,7 @@ def get_classes(module): return inspect.getmembers(module, inspect.isclass) -def get_plugins(pkg, baseclass): +def get_plugins(pkg, baseclass, path=None): """Return a list of all modules under a given package. - Modules must be a subclass of the provided 'baseclass' @@ -207,7 +212,7 @@ def get_plugins(pkg, baseclass): """ plugins = [] - modules = get_modules(pkg) + modules = get_modules(pkg, path=path) # Iterate through each module in the package for mod in modules: diff --git a/InvenTree/plugin/plugin.py b/InvenTree/plugin/plugin.py index 38d4982f6e..b59b42bc3b 100644 --- a/InvenTree/plugin/plugin.py +++ b/InvenTree/plugin/plugin.py @@ -275,7 +275,11 @@ class InvenTreePlugin(MixinBase, MetaBase): """Path to the plugin.""" if self._is_package: return self.__module__ # pragma: no cover - return pathlib.Path(self.def_path).relative_to(settings.BASE_DIR) + + try: + return pathlib.Path(self.def_path).relative_to(settings.BASE_DIR) + except ValueError: + return pathlib.Path(self.def_path) @property def settings_url(self): diff --git a/InvenTree/plugin/registry.py b/InvenTree/plugin/registry.py index efed23f6bb..1ef37ddee2 100644 --- a/InvenTree/plugin/registry.py +++ b/InvenTree/plugin/registry.py @@ -7,9 +7,9 @@ import importlib import logging import os -import pathlib import subprocess from importlib import metadata, reload +from pathlib import Path from typing import OrderedDict from django.apps import apps @@ -187,6 +187,53 @@ class PluginsRegistry: logger.info('Finished reloading plugins') + def plugin_dirs(self): + """Construct a list of directories from where plugins can be loaded""" + + dirs = ['plugin.builtin', ] + + if settings.TESTING or settings.DEBUG: + # If in TEST or DEBUG mode, load plugins from the 'samples' directory + dirs.append('plugin.samples') + + if settings.TESTING: + custom_dirs = os.getenv('INVENTREE_PLUGIN_TEST_DIR', None) + else: + custom_dirs = os.getenv('INVENTREE_PLUGIN_DIR', None) + + # Load from user specified directories (unless in testing mode) + dirs.append('plugins') + + if custom_dirs is not None: + # Allow multiple plugin directories to be specified + for pd_text in custom_dirs.split(','): + pd = Path(pd_text.strip()).absolute() + + # Attempt to create the directory if it does not already exist + if not pd.exists(): + try: + pd.mkdir(exist_ok=True) + except Exception: + logger.error(f"Could not create plugin directory '{pd}'") + continue + + # Ensure the directory has an __init__.py file + init_filename = pd.joinpath('__init__.py') + + if not init_filename.exists(): + try: + init_filename.write_text("# InvenTree plugin directory\n") + except Exception: + logger.error(f"Could not create file '{init_filename}'") + continue + + if pd.exists() and pd.is_dir(): + # By this point, we have confirmed that the directory at least exists + logger.info(f"Added plugin directory: '{pd}'") + dirs.append(pd) + + return dirs + def collect_plugins(self): """Collect plugins from all possible ways of loading.""" if not settings.PLUGINS_ENABLED: @@ -196,8 +243,20 @@ class PluginsRegistry: self.plugin_modules = [] # clear # Collect plugins from paths - for plugin in settings.PLUGIN_DIRS: - modules = get_plugins(importlib.import_module(plugin), InvenTreePlugin) + for plugin in self.plugin_dirs(): + + logger.info(f"Loading plugins from directory '{plugin}'") + + parent_path = None + parent_obj = Path(plugin) + + # If a "path" is provided, some special handling is required + if parent_obj.name is not plugin and len(parent_obj.parts) > 1: + parent_path = parent_obj.parent + plugin = parent_obj.name + + modules = get_plugins(importlib.import_module(plugin), InvenTreePlugin, path=parent_path) + if modules: [self.plugin_modules.append(item) for item in modules] @@ -224,7 +283,7 @@ class PluginsRegistry: return True try: - output = str(subprocess.check_output(['pip', 'install', '-U', '-r', settings.PLUGIN_FILE], cwd=os.path.dirname(settings.BASE_DIR)), 'utf-8') + output = str(subprocess.check_output(['pip', 'install', '-U', '-r', settings.PLUGIN_FILE], cwd=settings.BASE_DIR.parent), 'utf-8') except subprocess.CalledProcessError as error: # pragma: no cover logger.error(f'Ran into error while trying to install plugins!\n{str(error)}') return False @@ -506,7 +565,7 @@ class PluginsRegistry: """ try: # for local path plugins - plugin_path = '.'.join(pathlib.Path(plugin.path).relative_to(settings.BASE_DIR).parts) + plugin_path = '.'.join(Path(plugin.path).relative_to(settings.BASE_DIR).parts) except ValueError: # pragma: no cover # plugin is shipped as package plugin_path = plugin.NAME diff --git a/InvenTree/plugin/serializers.py b/InvenTree/plugin/serializers.py index 28bc8f6fa7..267d88f462 100644 --- a/InvenTree/plugin/serializers.py +++ b/InvenTree/plugin/serializers.py @@ -1,6 +1,5 @@ """JSON serializers for plugin app.""" -import os import subprocess from django.conf import settings @@ -144,7 +143,7 @@ class PluginConfigInstallSerializer(serializers.Serializer): success = False # execute pypi try: - result = subprocess.check_output(command, cwd=os.path.dirname(settings.BASE_DIR)) + result = subprocess.check_output(command, cwd=settings.BASE_DIR.parent) ret['result'] = str(result, 'utf-8') ret['success'] = True success = True diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index 1b1e402746..e55053b1c1 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -3,6 +3,7 @@ import logging import os import shutil +from pathlib import Path from django.apps import AppConfig from django.conf import settings @@ -25,23 +26,21 @@ class ReportConfig(AppConfig): def create_default_reports(self, model, reports): """Copy defualt report files across to the media directory.""" # Source directory for report templates - src_dir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), + src_dir = Path(__file__).parent.joinpath( 'templates', 'report', ) # Destination directory - dst_dir = os.path.join( - settings.MEDIA_ROOT, + dst_dir = settings.MEDIA_ROOT.joinpath( 'report', 'inventree', model.getSubdir(), ) - if not os.path.exists(dst_dir): + if not dst_dir.exists(): logger.info(f"Creating missing directory: '{dst_dir}'") - os.makedirs(dst_dir, exist_ok=True) + dst_dir.mkdir(parents=True, exist_ok=True) # Copy each report template across (if required) for report in reports: @@ -54,10 +53,10 @@ class ReportConfig(AppConfig): report['file'], ) - src_file = os.path.join(src_dir, report['file']) - dst_file = os.path.join(settings.MEDIA_ROOT, filename) + src_file = src_dir.joinpath(report['file']) + dst_file = settings.MEDIA_ROOT.joinpath(filename) - if not os.path.exists(dst_file): + if not dst_file.exists(): logger.info(f"Copying test report template '{dst_file}'") shutil.copyfile(src_file, dst_file) diff --git a/InvenTree/report/models.py b/InvenTree/report/models.py index 39ca0f7cd7..ea454d581d 100644 --- a/InvenTree/report/models.py +++ b/InvenTree/report/models.py @@ -111,14 +111,13 @@ class ReportBase(models.Model): path = os.path.join('report', 'report_template', self.getSubdir(), filename) - fullpath = os.path.join(settings.MEDIA_ROOT, path) - fullpath = os.path.abspath(fullpath) + fullpath = settings.MEDIA_ROOT.joinpath(path).resolve() # If the report file is the *same* filename as the one being uploaded, # remove the original one from the media directory if str(filename) == str(self.template): - if os.path.exists(fullpath): + if fullpath.exists(): logger.info(f"Deleting existing report template: '{filename}'") os.remove(fullpath) @@ -139,10 +138,12 @@ class ReportBase(models.Model): Required for passing the file to an external process """ template = self.template.name + + # TODO @matmair change to using new file objects template = template.replace('/', os.path.sep) template = template.replace('\\', os.path.sep) - template = os.path.join(settings.MEDIA_ROOT, template) + template = settings.MEDIA_ROOT.joinpath(template) return template @@ -474,14 +475,13 @@ def rename_snippet(instance, filename): path = os.path.join('report', 'snippets', filename) - fullpath = os.path.join(settings.MEDIA_ROOT, path) - fullpath = os.path.abspath(fullpath) + fullpath = settings.MEDIA_ROOT.joinpath(path).resolve() # If the snippet file is the *same* filename as the one being uploaded, # delete the original one from the media directory if str(filename) == str(instance.snippet): - if os.path.exists(fullpath): + if fullpath.exists(): logger.info(f"Deleting existing snippet file: '{filename}'") os.remove(fullpath) @@ -517,10 +517,9 @@ def rename_asset(instance, filename): # If the asset file is the *same* filename as the one being uploaded, # delete the original one from the media directory if str(filename) == str(instance.asset): - fullpath = os.path.join(settings.MEDIA_ROOT, path) - fullpath = os.path.abspath(fullpath) + fullpath = settings.MEDIA_ROOT.joinpath(path).resolve() - if os.path.exists(fullpath): + if fullpath.exists(): logger.info(f"Deleting existing asset file: '{filename}'") os.remove(fullpath) diff --git a/InvenTree/report/templatetags/report.py b/InvenTree/report/templatetags/report.py index 3b7863e2f4..cde637cfb4 100644 --- a/InvenTree/report/templatetags/report.py +++ b/InvenTree/report/templatetags/report.py @@ -32,9 +32,9 @@ def asset(filename): debug_mode = InvenTreeSetting.get_setting('REPORT_DEBUG_MODE') # Test if the file actually exists - full_path = os.path.join(settings.MEDIA_ROOT, 'report', 'assets', filename) + full_path = settings.MEDIA_ROOT.joinpath('report', 'assets', filename) - if not os.path.exists(full_path) or not os.path.isfile(full_path): + if not full_path.exists() or not full_path.is_file(): raise FileNotFoundError(f"Asset file '{filename}' does not exist") if debug_mode: @@ -63,9 +63,8 @@ def uploaded_image(filename, replace_missing=True, replacement_file='blank_image exists = False else: try: - full_path = os.path.join(settings.MEDIA_ROOT, filename) - full_path = os.path.abspath(full_path) - exists = os.path.exists(full_path) and os.path.isfile(full_path) + full_path = settings.MEDIA_ROOT.joinpath(filename).resolve() + exists = full_path.exists() and full_path.is_file() except Exception: exists = False @@ -85,11 +84,9 @@ def uploaded_image(filename, replace_missing=True, replacement_file='blank_image else: # Return file path if exists: - path = os.path.join(settings.MEDIA_ROOT, filename) - path = os.path.abspath(path) + path = settings.MEDIA_ROOT.joinpath(filename).resolve() else: - path = os.path.join(settings.STATIC_ROOT, 'img', replacement_file) - path = os.path.abspath(path) + path = settings.STATIC_ROOT.joinpath('img', replacement_file).resolve() return f"file://{path}" diff --git a/InvenTree/report/tests.py b/InvenTree/report/tests.py index f9e4c070fb..fe45b56446 100644 --- a/InvenTree/report/tests.py +++ b/InvenTree/report/tests.py @@ -2,6 +2,7 @@ import os import shutil +from pathlib import Path from django.conf import settings from django.core.cache import cache @@ -38,12 +39,11 @@ class ReportTagTest(TestCase): report_tags.asset("bad_file.txt") # Create an asset file - asset_dir = os.path.join(settings.MEDIA_ROOT, 'report', 'assets') - os.makedirs(asset_dir, exist_ok=True) - asset_path = os.path.join(asset_dir, 'test.txt') + asset_dir = settings.MEDIA_ROOT.joinpath('report', 'assets') + asset_dir.mkdir(parents=True, exist_ok=True) + asset_path = asset_dir.joinpath('test.txt') - with open(asset_path, 'w') as f: - f.write("dummy data") + asset_path.write_text("dummy data") self.debug_mode(True) asset = report_tags.asset('test.txt') @@ -68,13 +68,11 @@ class ReportTagTest(TestCase): # Create a dummy image img_path = 'part/images/' - img_path = os.path.join(settings.MEDIA_ROOT, img_path) - img_file = os.path.join(img_path, 'test.jpg') + img_path = settings.MEDIA_ROOT.joinpath(img_path) + img_file = img_path.joinpath('test.jpg') - os.makedirs(img_path, exist_ok=True) - - with open(img_file, 'w') as f: - f.write("dummy data") + img_path.mkdir(parents=True, exist_ok=True) + img_file.write_text("dummy data") # Test in debug mode. Returns blank image as dummy file is not a valid image self.debug_mode(True) @@ -91,7 +89,7 @@ class ReportTagTest(TestCase): self.debug_mode(False) img = report_tags.uploaded_image('part/images/test.jpg') - self.assertEqual(img, f'file://{img_path}test.jpg') + self.assertEqual(img, f'file://{img_path.joinpath("test.jpg")}') def test_part_image(self): """Unit tests for the 'part_image' tag""" @@ -178,8 +176,7 @@ class ReportTest(InvenTreeAPITestCase): def copyReportTemplate(self, filename, description): """Copy the provided report template into the required media directory.""" - src_dir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), + src_dir = Path(__file__).parent.joinpath( 'templates', 'report' ) @@ -190,18 +187,15 @@ class ReportTest(InvenTreeAPITestCase): self.model.getSubdir(), ) - dst_dir = os.path.join( - settings.MEDIA_ROOT, - template_dir - ) + dst_dir = settings.MEDIA_ROOT.joinpath(template_dir) - if not os.path.exists(dst_dir): # pragma: no cover - os.makedirs(dst_dir, exist_ok=True) + if not dst_dir.exists(): # pragma: no cover + dst_dir.mkdir(parents=True, exist_ok=True) - src_file = os.path.join(src_dir, filename) - dst_file = os.path.join(dst_dir, filename) + src_file = src_dir.joinpath(filename) + dst_file = dst_dir.joinpath(filename) - if not os.path.exists(dst_file): # pragma: no cover + if not dst_file.exists(): # pragma: no cover shutil.copyfile(src_file, dst_file) # Convert to an "internal" filename diff --git a/InvenTree/templates/InvenTree/settings/user.html b/InvenTree/templates/InvenTree/settings/user.html index 948c84bcac..07d0dc77f1 100644 --- a/InvenTree/templates/InvenTree/settings/user.html +++ b/InvenTree/templates/InvenTree/settings/user.html @@ -248,7 +248,11 @@ {% for object in session_list %} {{ object.ip }} + {% if object.user_agent or object.device %} {{ object.user_agent|device|default_if_none:unknown_on_unknown|safe }} + {% else %} + {{ unknown_on_unknown }} + {% endif %} {% if object.session_key == session_key %} {% blocktrans with time=object.last_activity|timesince %}{{ time }} ago (this session){% endblocktrans %} diff --git a/docker-compose.yml b/docker-compose.yml index 8ab381a5bd..1bb2867523 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,17 +25,17 @@ services: expose: - ${INVENTREE_DB_PORT:-5432}/tcp environment: - - PGDATA=/var/lib/postgresql/data/dev/pgdb + - PGDATA=/var/lib/postgresql/data/pgdb - POSTGRES_USER=${INVENTREE_DB_USER:?You must provide the 'INVENTREE_DB_USER' variable in the .env file} - POSTGRES_PASSWORD=${INVENTREE_DB_PASSWORD:?You must provide the 'INVENTREE_DB_PASSWORD' variable in the .env file} - POSTGRES_DB=${INVENTREE_DB_NAME:?You must provide the 'INVENTREE_DB_NAME' variable in the .env file} volumes: # Map 'data' volume such that postgres database is stored externally - - inventree_src:/var/lib/postgresql/data + - ./data:/var/lib/postgresql/data restart: unless-stopped - # InvenTree web server services - # Uses gunicorn as the web server + # InvenTree web server service + # Runs the django built-in webserver application inventree-dev-server: container_name: inventree-dev-server depends_on: @@ -48,13 +48,9 @@ services: ports: # Expose web server on port 8000 - 8000:8000 - # Note: If using the inventree-dev-proxy container (see below), - # comment out the "ports" directive (above) and uncomment the "expose" directive - #expose: - # - 8000 volumes: - # Ensure you specify the location of the 'src' directory at the end of this file - - inventree_src:/home/inventree + # Mount local source directory to /home/inventree + - ./:/home/inventree env_file: - .env restart: unless-stopped @@ -67,38 +63,8 @@ services: depends_on: - inventree-dev-server volumes: - # Ensure you specify the location of the 'src' directory at the end of this file - - inventree_src:/home/inventree + # Mount local source directory to /home/inventree + - ./:/home/inventree env_file: - .env restart: unless-stopped - - ### Optional: Serve static and media files using nginx - ### Uncomment the following lines to enable nginx proxy for testing - ### Note: If enabling the proxy, change "ports" to "expose" for the inventree-dev-server container (above) - #inventree-dev-proxy: - # container_name: inventree-dev-proxy - # image: nginx:stable - # depends_on: - # - inventree-dev-server - # ports: - # # Change "8000" to the port that you want InvenTree web server to be available on - # - 8000:80 - # volumes: - # # Provide ./nginx.dev.conf file to the container - # # Refer to the provided example file as a starting point - # - ./nginx.dev.conf:/etc/nginx/conf.d/default.conf:ro - # # nginx proxy needs access to static and media files - # - inventree_src:/var/www - # restart: unless-stopped - -volumes: - # Persistent data, stored external to the container(s) - inventree_src: - driver: local - driver_opts: - type: none - o: bind - # This directory specified where InvenTree source code is stored "outside" the docker containers - # By default, this directory is one level above the "docker" directory - device: ${INVENTREE_EXT_VOLUME:-./} diff --git a/docker/production/.env b/docker/production/.env index 171f2053e6..f61e799362 100644 --- a/docker/production/.env +++ b/docker/production/.env @@ -34,10 +34,12 @@ INVENTREE_DB_PORT=5432 #INVENTREE_DB_USER=pguser #INVENTREE_DB_PASSWORD=pgpassword -# Redis cache setup +# Redis cache setup (disabled by default) +# Un-comment the following lines to enable Redis cache +# Note that you will also have to run docker-compose with the --profile redis command # Refer to settings.py for other cache options -INVENTREE_CACHE_HOST=inventree-cache -INVENTREE_CACHE_PORT=6379 +#INVENTREE_CACHE_HOST=inventree-cache +#INVENTREE_CACHE_PORT=6379 # Enable plugins? INVENTREE_PLUGINS_ENABLED=False diff --git a/docker/production/docker-compose.yml b/docker/production/docker-compose.yml index 45f0c01c7b..e691ed7fa2 100644 --- a/docker/production/docker-compose.yml +++ b/docker/production/docker-compose.yml @@ -1,11 +1,11 @@ version: "3.8" -# Docker compose recipe for InvenTree production server, with the following containerized processes +# Docker compose recipe for a production-ready InvenTree setup, with the following containers: # - PostgreSQL as the database backend # - gunicorn as the InvenTree web server # - django-q as the InvenTree background worker process # - nginx as a reverse proxy -# - redis as the cache manager +# - redis as the cache manager (optional, disabled by default) # --------------------- # READ BEFORE STARTING! @@ -18,10 +18,8 @@ version: "3.8" # Changes made to this file are reflected across all containers! # # IMPORTANT NOTE: -# You should not have to change *anything* within the docker-compose.yml file! +# You should not have to change *anything* within this docker-compose.yml file! # Instead, make any changes in the .env file! -# The only *mandatory* change is to set the INVENTREE_EXT_VOLUME variable, -# which defines the directory (on your local machine) where persistent data are stored. # ------------------------ # InvenTree Image Versions @@ -29,15 +27,12 @@ version: "3.8" # By default, this docker-compose script targets the STABLE version of InvenTree, # image: inventree/inventree:stable # -# To run the LATEST (development) version of InvenTree, change the target image to: -# image: inventree/inventree:latest +# To run the LATEST (development) version of InvenTree, +# change the INVENTREE_TAG variable (in the .env file) to "latest" # # Alternatively, you could target a specific tagged release version with (for example): -# image: inventree/inventree:0.5.3 +# INVENTREE_TAG=0.7.5 # -# NOTE: If you change the target image, ensure it is the same for the following containers: -# - inventree-server -# - inventree-worker services: # Database service @@ -58,18 +53,21 @@ services: restart: unless-stopped # redis acts as database cache manager + # only runs under the "redis" profile : https://docs.docker.com/compose/profiles/ inventree-cache: container_name: inventree-cache image: redis:7.0 depends_on: - inventree-db + profiles: + - redis env_file: - .env expose: - ${INVENTREE_CACHE_PORT:-6379} restart: always - # InvenTree web server services + # InvenTree web server service # Uses gunicorn as the web server inventree-server: container_name: inventree-server @@ -79,13 +77,11 @@ services: - 8000 depends_on: - inventree-db - - inventree-cache env_file: - .env volumes: # Data volume must map to /home/inventree/data - inventree_data:/home/inventree/data - - inventree_plugins:/home/inventree/InvenTree/plugins restart: unless-stopped # Background worker process handles long-running or periodic tasks @@ -101,7 +97,6 @@ services: volumes: # Data volume must map to /home/inventree/data - inventree_data:/home/inventree/data - - inventree_plugins:/home/inventree/InvenTree/plugins restart: unless-stopped # nginx acts as a reverse proxy @@ -136,10 +131,3 @@ volumes: o: bind # This directory specified where InvenTree data are stored "outside" the docker containers device: ${INVENTREE_EXT_VOLUME:?You must specify the 'INVENTREE_EXT_VOLUME' variable in the .env file!} - inventree_plugins: - driver: local - driver_opts: - type: none - o: bind - # This directory specified where the optional local plugin directory is stored "outside" the docker containers - device: ${INVENTREE_EXT_PLUGINS:-./} diff --git a/tasks.py b/tasks.py index 87de041a8b..f131847f5c 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,7 @@ import os import pathlib import re import sys +from pathlib import Path from invoke import task @@ -52,23 +53,23 @@ def content_excludes(): return output -def localDir(): +def localDir() -> Path: """Returns the directory of *THIS* file. Used to ensure that the various scripts always run in the correct directory. """ - return os.path.dirname(os.path.abspath(__file__)) + return Path(__file__).parent.resolve() def managePyDir(): """Returns the directory of the manage.py file.""" - return os.path.join(localDir(), 'InvenTree') + return localDir().joinpath('InvenTree') def managePyPath(): """Return the path of the manage.py file.""" - return os.path.join(managePyDir(), 'manage.py') + return managePyDir().joinpath('manage.py') def manage(c, cmd, pty: bool = False): @@ -171,7 +172,7 @@ def translate_stats(c): The file generated from this is needed for the UI. """ - path = os.path.join('InvenTree', 'script', 'translation_stats.py') + path = Path('InvenTree', 'script', 'translation_stats.py') c.run(f'python3 {path}') @@ -252,12 +253,11 @@ def export_records(c, filename='data.json', overwrite=False, include_permissions """ # Get an absolute path to the file if not os.path.isabs(filename): - filename = os.path.join(localDir(), filename) - filename = os.path.abspath(filename) + filename = localDir().joinpath(filename).resolve() print(f"Exporting database records to file '{filename}'") - if os.path.exists(filename) and overwrite is False: + if filename.exists() and overwrite is False: response = input("Warning: file already exists. Do you want to overwrite? [y/N]: ") response = str(response).strip().lower() @@ -306,7 +306,7 @@ def import_records(c, filename='data.json', clear=False): """Import database records from a file.""" # Get an absolute path to the supplied filename if not os.path.isabs(filename): - filename = os.path.join(localDir(), filename) + filename = localDir().joinpath(filename) if not os.path.exists(filename): print(f"Error: File '{filename}' does not exist") @@ -442,8 +442,8 @@ def test_translations(c): from django.conf import settings # setup django - base_path = os.getcwd() - new_base_path = pathlib.Path('InvenTree').absolute() + base_path = Path.cwd() + new_base_path = pathlib.Path('InvenTree').resolve() sys.path.append(str(new_base_path)) os.chdir(new_base_path) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings') @@ -487,8 +487,8 @@ def test_translations(c): file_new.write(line) # change out translation files - os.rename(file_path, str(file_path) + '_old') - os.rename(new_file_path, file_path) + file_path.rename(str(file_path) + '_old') + new_file_path.rename(file_path) # compile languages print("Compile languages ...")