From 073a275d89ea53f8fdb4643222d949bfe0f9a3cb Mon Sep 17 00:00:00 2001 From: Lukas <76838159+wolflu05@users.noreply.github.com> Date: Wed, 26 Jul 2023 00:33:13 +0200 Subject: [PATCH] Skip ready functions if not in main thread or plugins are not loaded yet (#5005) * Skip ready functions if not in main thread or plugins are not loaded yet * Debug integration tests * Update ready.py * Update ready.py * Fix isInMainThread and isPluginRegistryLoaded ready functions * Preload gunicorn app to only invoke the appconfig ready functions once * debug: test prints for statistics * Remove debug print * Test without * Revert "Test without" This reverts commit 1bc18728935f2cebed38ae64ffe9f4e2f1d8e539. * Second test * Add checks back to part, label, user model * Add checks back to inventree, plugin apps * log server output for debugging * hopefully I can get the log this time+ * Next test * Test with --noreload * Next test * trigger: ci, because session expired * block the second ready execution instead of the first * fix: load order * Fix test and revert gh actions workflow change * Added all_apps method to reload machanism * Changed detect reload mechanism * Also trigger ready on reload * Add skipping second reload back for testing mode * Added doc string back * Update InvenTree/plugin/base/integration/AppMixin.py --- InvenTree/InvenTree/apps.py | 7 ++++- InvenTree/InvenTree/ready.py | 29 +++++++++++++++++++ InvenTree/InvenTree/settings.py | 2 +- InvenTree/common/tests.py | 2 +- InvenTree/gunicorn.conf.py | 3 ++ InvenTree/label/apps.py | 7 ++++- InvenTree/part/apps.py | 7 ++++- InvenTree/plugin/apps.py | 6 +++- InvenTree/plugin/base/integration/AppMixin.py | 6 +++- InvenTree/plugin/registry.py | 12 ++++++-- InvenTree/report/apps.py | 7 ++++- InvenTree/users/apps.py | 8 ++++- docker/gunicorn.conf.py | 3 ++ 13 files changed, 87 insertions(+), 12 deletions(-) diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index 2c553149b0..3b96266a8b 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -14,7 +14,8 @@ from django.db.utils import IntegrityError import InvenTree.conversion import InvenTree.tasks from InvenTree.config import get_setting -from InvenTree.ready import canAppAccessDatabase, isInTestMode +from InvenTree.ready import (canAppAccessDatabase, isInMainThread, + isInTestMode, isPluginRegistryLoaded) logger = logging.getLogger("inventree") @@ -34,6 +35,10 @@ class InvenTreeConfig(AppConfig): - Collecting notification methods - Adding users set in the current environment """ + # skip loading if plugin registry is not loaded or we run in a background thread + if not isPluginRegistryLoaded() or not isInMainThread(): + return + if canAppAccessDatabase() or settings.TESTING_ENV: InvenTree.tasks.check_for_migrations(worker=False) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index 6e16a3b471..cc7c690965 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -1,5 +1,6 @@ """Functions to check if certain parts of InvenTree are ready.""" +import os import sys @@ -18,6 +19,18 @@ def isRunningMigrations(): return 'migrate' in sys.argv or 'makemigrations' in sys.argv +def isInMainThread(): + """Django runserver starts two processes, one for the actual dev server and the other to reload the application. + + - The RUN_MAIN env is set in that case. However if --noreload is applied, this variable + is not set because there are no different threads. + """ + if "runserver" in sys.argv and "--noreload" not in sys.argv: + return os.environ.get('RUN_MAIN', None) == "true" + + return True + + def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False, allow_shell: bool = False): """Returns True if the apps.py file can access database records. @@ -65,3 +78,19 @@ def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False, return False return True + + +def isPluginRegistryLoaded(): + """Ensures that the plugin registry is already loaded. + + The plugin registry reloads all apps onetime after starting if there are AppMixin plugins, + so that the discovered AppConfigs are added to Django. This triggers the ready function of + AppConfig to execute twice. Add this check to prevent from running two times. + + Note: All apps using this check need to be registered after the plugins app in settings.py + + Returns: 'False' if the registry has not fully loaded the plugins yet. + """ + from plugin import registry + + return registry.plugins_loaded diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index edfef7284b..8323c2e0a7 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -199,13 +199,13 @@ INSTALLED_APPS = [ 'build.apps.BuildConfig', 'common.apps.CommonConfig', 'company.apps.CompanyConfig', + 'plugin.apps.PluginAppConfig', # Plugin app runs before all apps that depend on the isPluginRegistryLoaded function 'label.apps.LabelConfig', 'order.apps.OrderConfig', 'part.apps.PartConfig', 'report.apps.ReportConfig', 'stock.apps.StockConfig', 'users.apps.UsersConfig', - 'plugin.apps.PluginAppConfig', 'web', 'generic', 'InvenTree.apps.InvenTreeConfig', # InvenTree app runs last diff --git a/InvenTree/common/tests.py b/InvenTree/common/tests.py index d9de170d07..0a0f9bdb7e 100644 --- a/InvenTree/common/tests.py +++ b/InvenTree/common/tests.py @@ -879,7 +879,7 @@ class CommonTest(InvenTreeAPITestCase): from plugin import registry # set flag true - common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', False, None) + common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', True, None) # reload the app registry.reload_plugins() diff --git a/InvenTree/gunicorn.conf.py b/InvenTree/gunicorn.conf.py index b11999aadd..c1b403de12 100644 --- a/InvenTree/gunicorn.conf.py +++ b/InvenTree/gunicorn.conf.py @@ -8,3 +8,6 @@ workers = multiprocessing.cpu_count() * 2 + 1 max_requests = 1000 max_requests_jitter = 50 + +# preload app so that the ready functions are only executed once +preload_app = True diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index 850e55c355..270de65b56 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -12,7 +12,8 @@ from django.conf import settings from django.core.exceptions import AppRegistryNotReady from django.db.utils import OperationalError -from InvenTree.ready import canAppAccessDatabase +from InvenTree.ready import (canAppAccessDatabase, isInMainThread, + isPluginRegistryLoaded) logger = logging.getLogger("inventree") @@ -35,6 +36,10 @@ class LabelConfig(AppConfig): def ready(self): """This function is called whenever the label app is loaded.""" + # skip loading if plugin registry is not loaded or we run in a background thread + if not isPluginRegistryLoaded() or not isInMainThread(): + return + if canAppAccessDatabase(allow_test=False): try: diff --git a/InvenTree/part/apps.py b/InvenTree/part/apps.py index 298ff486cd..73cd429146 100644 --- a/InvenTree/part/apps.py +++ b/InvenTree/part/apps.py @@ -5,7 +5,8 @@ import logging from django.apps import AppConfig from django.db.utils import OperationalError, ProgrammingError -from InvenTree.ready import canAppAccessDatabase, isImportingData +from InvenTree.ready import (canAppAccessDatabase, isImportingData, + isInMainThread, isPluginRegistryLoaded) logger = logging.getLogger("inventree") @@ -16,6 +17,10 @@ class PartConfig(AppConfig): def ready(self): """This function is called whenever the Part app is loaded.""" + # skip loading if plugin registry is not loaded or we run in a background thread + if not isPluginRegistryLoaded() or not isInMainThread(): + return + if canAppAccessDatabase(): self.update_trackable_status() self.reset_part_pricing_flags() diff --git a/InvenTree/plugin/apps.py b/InvenTree/plugin/apps.py index b3b1a4cb6a..ca26228653 100644 --- a/InvenTree/plugin/apps.py +++ b/InvenTree/plugin/apps.py @@ -10,7 +10,7 @@ from django.apps import AppConfig from maintenance_mode.core import set_maintenance_mode -from InvenTree.ready import canAppAccessDatabase +from InvenTree.ready import canAppAccessDatabase, isInMainThread from plugin import registry logger = logging.getLogger('inventree') @@ -23,6 +23,10 @@ class PluginAppConfig(AppConfig): def ready(self): """The ready method is extended to initialize plugins.""" + # skip loading if we run in a background thread + if not isInMainThread(): + return + if not canAppAccessDatabase(allow_test=True, allow_plugins=True): logger.info("Skipping plugin loading sequence") # pragma: no cover else: diff --git a/InvenTree/plugin/base/integration/AppMixin.py b/InvenTree/plugin/base/integration/AppMixin.py index cb33ab28b3..4334dabaaf 100644 --- a/InvenTree/plugin/base/integration/AppMixin.py +++ b/InvenTree/plugin/base/integration/AppMixin.py @@ -46,8 +46,12 @@ class AppMixin: settings.INSTALLED_APPS += [plugin_path] registry.installed_apps += [plugin_path] apps_changed = True + # if apps were changed or force loading base apps -> reload - if apps_changed or force_reload: + # Ignore reloading if we are in testing mode and apps are unchanged so that tests run faster + # registry.reload_plugins(...) first unloads and then loads the plugins + # always reload if we are not in testing mode so we can expect the second reload + if not settings.TESTING or apps_changed or force_reload: # first startup or force loading of base apps -> registry is prob false if registry.apps_loading or force_reload: registry.apps_loading = False diff --git a/InvenTree/plugin/registry.py b/InvenTree/plugin/registry.py index 5ddd9857af..3fd402d471 100644 --- a/InvenTree/plugin/registry.py +++ b/InvenTree/plugin/registry.py @@ -11,7 +11,7 @@ import os import subprocess import time from pathlib import Path -from typing import Dict, List, OrderedDict +from typing import Any, Dict, List, OrderedDict from django.apps import apps from django.conf import settings @@ -54,13 +54,14 @@ class PluginsRegistry: self.plugins_inactive: Dict[str, InvenTreePlugin] = {} # List of inactive instances self.plugins_full: Dict[str, InvenTreePlugin] = {} # List of all plugin instances - self.plugin_modules: List(InvenTreePlugin) = [] # Holds all discovered plugins - self.mixin_modules: Dict[str, any] = {} # Holds all discovered mixins + self.plugin_modules: List[InvenTreePlugin] = [] # Holds all discovered plugins + self.mixin_modules: Dict[str, Any] = {} # Holds all discovered mixins self.errors = {} # Holds discovering errors # flags self.is_loading = False # Are plugins being loaded right now + self.plugins_loaded = False # Marks if the registry fully loaded and all django apps are reloaded self.apps_loading = True # Marks if apps were reloaded yet self.installed_apps = [] # Holds all added plugin_paths @@ -161,6 +162,9 @@ class PluginsRegistry: if full_reload: full_reload = False + # ensure plugins_loaded is True + self.plugins_loaded = True + # Remove maintenance mode if not _maintenance: set_maintenance_mode(False) @@ -212,7 +216,9 @@ class PluginsRegistry: logger.info('Start reloading plugins') with maintenance_mode_on(): + self.plugins_loaded = False self.unload_plugins(force_reload=force_reload) + self.plugins_loaded = True self.load_plugins(full_reload=full_reload) logger.info('Finished reloading plugins') diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index 74999d1cae..7c64c8609c 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -18,7 +18,12 @@ class ReportConfig(AppConfig): def ready(self): """This function is called whenever the report app is loaded.""" - from InvenTree.ready import canAppAccessDatabase + from InvenTree.ready import (canAppAccessDatabase, isInMainThread, + isPluginRegistryLoaded) + + # skip loading if plugin registry is not loaded or we run in a background thread + if not isPluginRegistryLoaded() or not isInMainThread(): + return # Configure logging for PDF generation (disable "info" messages) logging.getLogger('fontTools').setLevel(logging.WARNING) diff --git a/InvenTree/users/apps.py b/InvenTree/users/apps.py index 71c2d079ce..ccaa57c1dc 100644 --- a/InvenTree/users/apps.py +++ b/InvenTree/users/apps.py @@ -5,7 +5,8 @@ import logging from django.apps import AppConfig from django.db.utils import OperationalError, ProgrammingError -from InvenTree.ready import canAppAccessDatabase +from InvenTree.ready import (canAppAccessDatabase, isInMainThread, + isPluginRegistryLoaded) logger = logging.getLogger('inventree') @@ -17,6 +18,11 @@ class UsersConfig(AppConfig): def ready(self): """Called when the 'users' app is loaded at runtime""" + + # skip loading if plugin registry is not loaded or we run in a background thread + if not isPluginRegistryLoaded() or not isInMainThread(): + return + if canAppAccessDatabase(allow_test=True): try: diff --git a/docker/gunicorn.conf.py b/docker/gunicorn.conf.py index 750f293380..58b48a0c8d 100644 --- a/docker/gunicorn.conf.py +++ b/docker/gunicorn.conf.py @@ -37,3 +37,6 @@ logger.info(f"Starting gunicorn server with {workers} workers") max_requests = 1000 max_requests_jitter = 50 + +# preload app so that the ready functions are only executed once +preload_app = True