From 1b76828305ec9f3160bf01fcac460ad5dcc58540 Mon Sep 17 00:00:00 2001 From: Matthias Mair Date: Tue, 16 Aug 2022 08:10:18 +0200 Subject: [PATCH] Plugin framework updates (#3535) * refactor entrypoint into helpers * Add lookup by metadata This is geared towards plugins packaged in pkgs that differ in name from their top module * Make module lookup predictable in changing pkg-envs * remove no coverage from plugin packages * ignore coverage for production loadin * refactor plugin collection - move assigment out * do not cover fs errors * test custom dir loading * test module meta fetcher * add a bit more safety * do not cover sanity checkers * add folder loading test * ignore again for cleaner diffs for now * ignore safety catch * rename test * Add test for package installs * fix docstring name * depreciate test for now * Fix for out of BASE_DIR paths * ignore catch * remove unneeded complexity * add testing for outside folders * more docstrings and simpler methods * make call simpler * refactor import * Add registry with all plugins * use full registry and make simpler request * switch path properties to methods * Add typing to plugin * Add a checker fnc for is_sample * Add sample check to admin page * Make file check a cls * more cls methods * Add setting for signature cheks Fixes #3520 * make property statements simpler * use same key in all dicts * Use module name instead of NAME Fixes #3534 * fix naming * fix name * add version checking Fixes #3478 * fix formatting and typing * also save reference to full array * do not cover as we turn on all plugins * add test for check_version * Add version e2e test * make test save * refactor out assignment * safe a db reference first * docstring * condense code a bit * rename * append logging * rename * also safe db reference to new object * docstrings, refactors, typing * fix key lookup --- InvenTree/common/models.py | 7 + InvenTree/plugin/admin.py | 2 +- InvenTree/plugin/models.py | 24 ++- InvenTree/plugin/plugin.py | 80 +++++++--- InvenTree/plugin/registry.py | 147 +++++++++--------- .../plugin/samples/integration/version.py | 9 ++ InvenTree/plugin/template.py | 6 +- InvenTree/plugin/test_plugin.py | 18 +++ .../templates/InvenTree/settings/plugin.html | 2 + .../InvenTree/settings/plugin_settings.html | 4 + 10 files changed, 197 insertions(+), 102 deletions(-) create mode 100644 InvenTree/plugin/samples/integration/version.py diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index a006dab47b..a0752e4915 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -1268,6 +1268,13 @@ class InvenTreeSetting(BaseInvenTreeSetting): 'requires_restart': True, }, + 'PLUGIN_CHECK_SIGNATURES': { + 'name': _('Check plugin signatures'), + 'description': _('Check and show signatures for plugins'), + 'default': False, + 'validator': bool, + }, + # Settings for plugin mixin features 'ENABLE_PLUGINS_URL': { 'name': _('Enable URL integration'), diff --git a/InvenTree/plugin/admin.py b/InvenTree/plugin/admin.py index 84da9e9d98..7bb0558600 100644 --- a/InvenTree/plugin/admin.py +++ b/InvenTree/plugin/admin.py @@ -52,7 +52,7 @@ class PluginConfigAdmin(admin.ModelAdmin): """Custom admin with restricted id fields.""" readonly_fields = ["key", "name", ] - list_display = ['name', 'key', '__str__', 'active', ] + list_display = ['name', 'key', '__str__', 'active', 'is_sample'] list_filter = ['active'] actions = [plugin_activate, plugin_deactivate, ] inlines = [PluginSettingInline, ] diff --git a/InvenTree/plugin/models.py b/InvenTree/plugin/models.py index 9a58e96fdc..16e51513da 100644 --- a/InvenTree/plugin/models.py +++ b/InvenTree/plugin/models.py @@ -3,6 +3,7 @@ import warnings from django.conf import settings +from django.contrib import admin from django.contrib.auth.models import User from django.db import models from django.utils.translation import gettext_lazy as _ @@ -123,11 +124,11 @@ class PluginConfig(models.Model): self.__org_active = self.active # append settings from registry - self.plugin = registry.plugins.get(self.key, None) + plugin = registry.plugins_full.get(self.key, None) def get_plugin_meta(name): - if self.plugin: - return str(getattr(self.plugin, name, None)) + if plugin: + return str(getattr(plugin, name, None)) return None self.meta = { @@ -136,6 +137,9 @@ class PluginConfig(models.Model): 'package_path', 'settings_url', ] } + # Save plugin + self.plugin: InvenTreePlugin = plugin + def save(self, force_insert=False, force_update=False, *args, **kwargs): """Extend save method to reload plugins if the 'active' status changes.""" reload = kwargs.pop('no_reload', False) # check if no_reload flag is set @@ -151,6 +155,20 @@ class PluginConfig(models.Model): return ret + @admin.display(boolean=True, description=_('Sample plugin')) + def is_sample(self) -> bool: + """Is this plugin a sample app?""" + # Loaded and active plugin + if isinstance(self.plugin, InvenTreePlugin): + return self.plugin.check_is_sample() + + # If no plugin_class is available it can not be a sample + if not self.plugin: + return False + + # Not loaded plugin + return self.plugin.check_is_sample() # pragma: no cover + class PluginSetting(common.models.BaseInvenTreeSetting): """This model represents settings for individual plugins.""" diff --git a/InvenTree/plugin/plugin.py b/InvenTree/plugin/plugin.py index 2bf849c56e..f23e34107f 100644 --- a/InvenTree/plugin/plugin.py +++ b/InvenTree/plugin/plugin.py @@ -2,11 +2,10 @@ import inspect import logging -import os -import pathlib import warnings from datetime import datetime from importlib.metadata import PackageNotFoundError, metadata +from pathlib import Path from django.conf import settings from django.db.utils import OperationalError, ProgrammingError @@ -171,7 +170,24 @@ class MixinBase: return mixins -class InvenTreePlugin(MixinBase, MetaBase): +class VersionMixin: + """Mixin to enable version checking.""" + + MIN_VERSION = None + MAX_VERSION = None + + def check_version(self, latest=None) -> bool: + """Check if plugin functions for the current InvenTree version.""" + from InvenTree import version + + latest = latest if latest else version.inventreeVersionTuple() + min_v = version.inventreeVersionTuple(self.MIN_VERSION) + max_v = version.inventreeVersionTuple(self.MAX_VERSION) + + return bool(min_v <= latest <= max_v) + + +class InvenTreePlugin(VersionMixin, MixinBase, MetaBase): """The InvenTreePlugin class is used to integrate with 3rd party software. DO NOT USE THIS DIRECTLY, USE plugin.InvenTreePlugin @@ -191,11 +207,18 @@ class InvenTreePlugin(MixinBase, MetaBase): """ super().__init__() self.add_mixin('base') - self.def_path = inspect.getfile(self.__class__) - self.path = os.path.dirname(self.def_path) self.define_package() + @classmethod + def file(cls) -> Path: + """File that contains plugin definition.""" + return Path(inspect.getfile(cls)) + + def path(self) -> Path: + """Path to plugins base folder.""" + return self.file().parent + def _get_value(self, meta_name: str, package_name: str) -> str: """Extract values from class meta or package info. @@ -243,43 +266,54 @@ class InvenTreePlugin(MixinBase, MetaBase): @property def version(self): """Version of plugin.""" - version = self._get_value('VERSION', 'version') - return version + return self._get_value('VERSION', 'version') @property def website(self): """Website of plugin - if set else None.""" - website = self._get_value('WEBSITE', 'website') - return website + return self._get_value('WEBSITE', 'website') @property def license(self): """License of plugin.""" - lic = self._get_value('LICENSE', 'license') - return lic + return self._get_value('LICENSE', 'license') # endregion + @classmethod + def check_is_package(cls): + """Is the plugin delivered as a package.""" + return getattr(cls, 'is_package', False) + @property def _is_package(self): """Is the plugin delivered as a package.""" return getattr(self, 'is_package', False) - @property - def is_sample(self): + @classmethod + def check_is_sample(cls) -> bool: """Is this plugin part of the samples?""" - path = str(self.package_path) - return path.startswith('plugin/samples/') + return str(cls.check_package_path()).startswith('plugin/samples/') + + @property + def is_sample(self) -> bool: + """Is this plugin part of the samples?""" + return self.check_is_sample() + + @classmethod + def check_package_path(cls): + """Path to the plugin.""" + if cls.check_is_package(): + return cls.__module__ # pragma: no cover + + try: + return cls.file().relative_to(settings.BASE_DIR) + except ValueError: + return cls.file() @property def package_path(self): """Path to the plugin.""" - if self._is_package: - return self.__module__ # pragma: no cover - - try: - return pathlib.Path(self.def_path).relative_to(settings.BASE_DIR) - except ValueError: - return pathlib.Path(self.def_path) + return self.check_package_path() @property def settings_url(self): @@ -289,7 +323,7 @@ class InvenTreePlugin(MixinBase, MetaBase): # region package info def _get_package_commit(self): """Get last git commit for the plugin.""" - return get_git_log(self.def_path) + return get_git_log(str(self.file())) @classmethod def _get_package_metadata(cls): diff --git a/InvenTree/plugin/registry.py b/InvenTree/plugin/registry.py index ec0b13410c..9b0e8e29c0 100644 --- a/InvenTree/plugin/registry.py +++ b/InvenTree/plugin/registry.py @@ -11,7 +11,7 @@ import os import subprocess from importlib import reload from pathlib import Path -from typing import OrderedDict +from typing import Dict, List, OrderedDict from django.apps import apps from django.conf import settings @@ -41,19 +41,20 @@ class PluginsRegistry: Set up all needed references for internal and external states. """ # plugin registry - self.plugins = {} - self.plugins_inactive = {} + self.plugins: Dict[str, InvenTreePlugin] = {} # List of active instances + self.plugins_inactive: Dict[str, InvenTreePlugin] = {} # List of inactive instances + self.plugins_full: Dict[str, InvenTreePlugin] = {} # List of all plugin instances - self.plugin_modules = [] # Holds all discovered plugins + self.plugin_modules: List(InvenTreePlugin) = [] # Holds all discovered plugins - self.errors = {} # Holds discovering errors + self.errors = {} # Holds discovering errors # flags - self.is_loading = False - self.apps_loading = True # Marks if apps were reloaded yet - self.git_is_modern = True # Is a modern version of git available + self.is_loading = False # Are plugins beeing loaded right now + self.apps_loading = True # Marks if apps were reloaded yet + self.git_is_modern = True # Is a modern version of git available - self.installed_apps = [] # Holds all added plugin_paths + self.installed_apps = [] # Holds all added plugin_paths # mixins self.mixins_settings = {} @@ -339,74 +340,77 @@ class PluginsRegistry: # endregion # region general internal loading /activating / deactivating / deloading - def _init_plugins(self, disabled=None): + def _init_plugins(self, disabled: str = None): """Initialise all found plugins. - :param disabled: loading path of disabled app, defaults to None - :type disabled: str, optional - :raises error: IntegrationPluginError + Args: + disabled (str, optional): Loading path of disabled app. Defaults to None. + + Raises: + error: IntegrationPluginError """ from plugin.models import PluginConfig + def safe_reference(plugin, key: str, active: bool = True): + """Safe reference to plugin dicts.""" + if active: + self.plugins[key] = plugin + else: + # Deactivate plugin in db + if not settings.PLUGIN_TESTING: # pragma: no cover + plugin.db.active = False + plugin.db.save(no_reload=True) + self.plugins_inactive[key] = plugin.db + self.plugins_full[key] = plugin + logger.info('Starting plugin initialisation') # Initialize plugins - for plugin in self.plugin_modules: - # Check if package - was_packaged = getattr(plugin, 'is_package', False) - - # Check if activated + for plg in self.plugin_modules: # These checks only use attributes - never use plugin supplied functions -> that would lead to arbitrary code execution!! - plug_name = plugin.NAME - plug_key = plugin.SLUG if getattr(plugin, 'SLUG', None) else plug_name - plug_key = slugify(plug_key) # keys are slugs! + plg_name = plg.NAME + plg_key = slugify(plg.SLUG if getattr(plg, 'SLUG', None) else plg_name) # keys are slugs! + try: - plugin_db_setting, _ = PluginConfig.objects.get_or_create(key=plug_key, name=plug_name) + plg_db, _ = PluginConfig.objects.get_or_create(key=plg_key, name=plg_name) except (OperationalError, ProgrammingError) as error: # Exception if the database has not been migrated yet - check if test are running - raise if not if not settings.PLUGIN_TESTING: raise error # pragma: no cover - plugin_db_setting = None + plg_db = None except (IntegrityError) as error: # pragma: no cover - logger.error(f"Error initializing plugin: {error}") + logger.error(f"Error initializing plugin `{plg_name}`: {error}") + + # Append reference to plugin + plg.db = plg_db # Always activate if testing - if settings.PLUGIN_TESTING or (plugin_db_setting and plugin_db_setting.active): - # Check if the plugin was blocked -> threw an error - if disabled: - # option1: package, option2: file-based - if (plugin.__name__ == disabled) or (plugin.__module__ == disabled): - # Errors are bad so disable the plugin in the database - if not settings.PLUGIN_TESTING: # pragma: no cover - plugin_db_setting.active = False - plugin_db_setting.save(no_reload=True) - - # Add to inactive plugins so it shows up in the ui - self.plugins_inactive[plug_key] = plugin_db_setting - continue # continue -> the plugin is not loaded - - # Initialize package - # now we can be sure that an admin has activated the plugin - logger.info(f'Loading plugin {plug_name}') + if settings.PLUGIN_TESTING or (plg_db and plg_db.active): + # Check if the plugin was blocked -> threw an error; option1: package, option2: file-based + if disabled and ((plg.__name__ == disabled) or (plg.__module__ == disabled)): + safe_reference(plugin=plg, key=plg_key, active=False) + continue # continue -> the plugin is not loaded + # Initialize package - we can be sure that an admin has activated the plugin + logger.info(f'Loading plugin `{plg_name}`') try: - plugin = plugin() + plg_i: InvenTreePlugin = plg() + logger.info(f'Loaded plugin `{plg_name}`') except Exception as error: - # log error and raise it -> disable plugin - handle_error(error, log_name='init') + handle_error(error, log_name='init') # log error and raise it -> disable plugin - logger.debug(f'Loaded plugin {plug_name}') + # Safe extra attributes + plg_i.is_package = getattr(plg_i, 'is_package', False) + plg_i.pk = plg_db.pk if plg_db else None + plg_i.db = plg_db - plugin.is_package = was_packaged - - if plugin_db_setting: - plugin.pk = plugin_db_setting.pk - - # safe reference - self.plugins[plugin.slug] = plugin - else: - # save for later reference - self.plugins_inactive[plug_key] = plugin_db_setting # pragma: no cover + # Run version check for plugin + if (plg_i.MIN_VERSION or plg_i.MAX_VERSION) and not plg_i.check_version(): + safe_reference(plugin=plg_i, key=plg_key, active=False) + else: + safe_reference(plugin=plg_i, key=plg_key) + else: # pragma: no cover + safe_reference(plugin=plg, key=plg_key, active=False) def _activate_plugins(self, force_reload=False, full_reload: bool = False): """Run activation functions for all plugins. @@ -583,10 +587,10 @@ class PluginsRegistry: """ try: # for local path plugins - plugin_path = '.'.join(Path(plugin.path).relative_to(settings.BASE_DIR).parts) + plugin_path = '.'.join(plugin.path().relative_to(settings.BASE_DIR).parts) except ValueError: # pragma: no cover - # plugin is shipped as package - plugin_path = plugin.NAME + # plugin is shipped as package - extract plugin module name + plugin_path = plugin.__module__.split('.')[0] return plugin_path def deactivate_plugin_app(self): @@ -640,24 +644,25 @@ class PluginsRegistry: self.installed_apps = [] def _clean_registry(self): - # remove all plugins from registry - self.plugins = {} - self.plugins_inactive = {} + """Remove all plugins from registry.""" + self.plugins: Dict[str, InvenTreePlugin] = {} + self.plugins_inactive: Dict[str, InvenTreePlugin] = {} + self.plugins_full: Dict[str, InvenTreePlugin] = {} def _update_urls(self): - from InvenTree.urls import frontendpatterns as urlpatterns + from InvenTree.urls import frontendpatterns as urlpattern from InvenTree.urls import urlpatterns as global_pattern from plugin.urls import get_plugin_urls - for index, a in enumerate(urlpatterns): - if hasattr(a, 'app_name'): - if a.app_name == 'admin': - urlpatterns[index] = re_path(r'^admin/', admin.site.urls, name='inventree-admin') - elif a.app_name == 'plugin': - urlpatterns[index] = get_plugin_urls() + for index, url in enumerate(urlpattern): + if hasattr(url, 'app_name'): + if url.app_name == 'admin': + urlpattern[index] = re_path(r'^admin/', admin.site.urls, name='inventree-admin') + elif url.app_name == 'plugin': + urlpattern[index] = get_plugin_urls() - # replace frontendpatterns - global_pattern[0] = re_path('', include(urlpatterns)) + # Replace frontendpatterns + global_pattern[0] = re_path('', include(urlpattern)) clear_url_caches() def _reload_apps(self, force_reload: bool = False, full_reload: bool = False): @@ -693,7 +698,7 @@ class PluginsRegistry: # endregion -registry = PluginsRegistry() +registry: PluginsRegistry = PluginsRegistry() def call_function(plugin_name, function_name, *args, **kwargs): diff --git a/InvenTree/plugin/samples/integration/version.py b/InvenTree/plugin/samples/integration/version.py new file mode 100644 index 0000000000..47314f3df6 --- /dev/null +++ b/InvenTree/plugin/samples/integration/version.py @@ -0,0 +1,9 @@ +"""Sample plugin for versioning.""" +from plugin import InvenTreePlugin + + +class VersionPlugin(InvenTreePlugin): + """A small version sample.""" + + NAME = "version" + MAX_VERSION = '0.1.0' diff --git a/InvenTree/plugin/template.py b/InvenTree/plugin/template.py index c16135a7a3..01e57c1958 100644 --- a/InvenTree/plugin/template.py +++ b/InvenTree/plugin/template.py @@ -1,7 +1,5 @@ """Load templates for loaded plugins.""" -from pathlib import Path - from django.template.loaders.filesystem import Loader as FilesystemLoader from plugin import registry @@ -26,8 +24,8 @@ class PluginTemplateLoader(FilesystemLoader): template_dirs = [] for plugin in registry.plugins.values(): - new_path = Path(plugin.path) / dirname - if Path(new_path).is_dir(): + new_path = plugin.path().joinpath(dirname) + if new_path.is_dir(): template_dirs.append(new_path) return tuple(template_dirs) diff --git a/InvenTree/plugin/test_plugin.py b/InvenTree/plugin/test_plugin.py index 7dfdaf44ad..4df129bc0d 100644 --- a/InvenTree/plugin/test_plugin.py +++ b/InvenTree/plugin/test_plugin.py @@ -100,6 +100,14 @@ class InvenTreePluginTests(TestCase): self.plugin_name = NameInvenTreePlugin() self.plugin_sample = SampleIntegrationPlugin() + class VersionInvenTreePlugin(InvenTreePlugin): + NAME = 'Version' + + MIN_VERSION = '0.1.0' + MAX_VERSION = '0.1.3' + + self.plugin_version = VersionInvenTreePlugin() + def test_basic_plugin_init(self): """Check if a basic plugin intis.""" self.assertEqual(self.plugin.NAME, '') @@ -169,6 +177,16 @@ class InvenTreePluginTests(TestCase): # check default value is used self.assertEqual(self.plugin_old.get_meta_value('ABC', 'ABCD', '123'), '123') + def test_version(self): + """Test Version checks""" + + self.assertFalse(self.plugin_version.check_version([0, 0, 3])) + self.assertTrue(self.plugin_version.check_version([0, 1, 0])) + self.assertFalse(self.plugin_version.check_version([0, 1, 4])) + + plug = registry.plugins_full.get('version') + self.assertEqual(plug.is_active(), False) + class RegistryTests(TestCase): """Tests for registry loading methods.""" diff --git a/InvenTree/templates/InvenTree/settings/plugin.html b/InvenTree/templates/InvenTree/settings/plugin.html index 4e2310832b..8051563d24 100644 --- a/InvenTree/templates/InvenTree/settings/plugin.html +++ b/InvenTree/templates/InvenTree/settings/plugin.html @@ -25,6 +25,8 @@ {% include "InvenTree/settings/setting.html" with key="ENABLE_PLUGINS_NAVIGATION" icon="fa-sitemap" %} {% include "InvenTree/settings/setting.html" with key="ENABLE_PLUGINS_APP" icon="fa-rocket" %} {% include "InvenTree/settings/setting.html" with key="PLUGIN_ON_STARTUP" %} + + {% include "InvenTree/settings/setting.html" with key="PLUGIN_CHECK_SIGNATURES" %} diff --git a/InvenTree/templates/InvenTree/settings/plugin_settings.html b/InvenTree/templates/InvenTree/settings/plugin_settings.html index 2ba3a401b2..0d06ead680 100644 --- a/InvenTree/templates/InvenTree/settings/plugin_settings.html +++ b/InvenTree/templates/InvenTree/settings/plugin_settings.html @@ -112,6 +112,9 @@ {% trans "Commit Message" %}{{ plugin.package.message }}{% include "clip.html" %} {% endif %} + + {% settings_value "PLUGIN_CHECK_SIGNATURES" as signatures %} + {% if signatures %} {% trans "Sign Status" %} @@ -122,6 +125,7 @@ {% trans "Sign Key" %} {{ plugin.package.key }}{% include "clip.html" %} + {% endif %}