diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index bd4e039122..d40ea4eb29 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -12,10 +12,9 @@ from django.db import transaction from django.db.utils import IntegrityError import InvenTree.tasks +from InvenTree.config import get_setting from InvenTree.ready import canAppAccessDatabase, isInTestMode -from .config import get_setting - logger = logging.getLogger("inventree") @@ -24,8 +23,18 @@ class InvenTreeConfig(AppConfig): name = 'InvenTree' def ready(self): - """Setup background tasks and update exchange rates.""" + """Run system wide setup init steps. + + Like: + - Checking if migrations should be run + - Cleaning up tasks + - Starting regular tasks + - Updateing exchange rates + - Collecting notification mehods + - Adding users set in the current environment + """ if canAppAccessDatabase() or settings.TESTING_ENV: + InvenTree.tasks.check_for_migrations(worker=False) self.remove_obsolete_tasks() diff --git a/InvenTree/InvenTree/migrations/0001_initial.py b/InvenTree/InvenTree/migrations/0001_initial.py new file mode 100644 index 0000000000..29ca7615f7 --- /dev/null +++ b/InvenTree/InvenTree/migrations/0001_initial.py @@ -0,0 +1,12 @@ +# Generated by Django 3.2.15 on 2022-10-03 18:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + ] diff --git a/InvenTree/InvenTree/migrations/__init__.py b/InvenTree/InvenTree/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index f3f3b9dc1b..3e39076ca3 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -15,10 +15,17 @@ from django.conf import settings from django.core import mail as django_mail from django.core.exceptions import AppRegistryNotReady from django.core.management import call_command -from django.db.utils import OperationalError, ProgrammingError +from django.db import DEFAULT_DB_ALIAS, connections +from django.db.migrations.executor import MigrationExecutor +from django.db.utils import (NotSupportedError, OperationalError, + ProgrammingError) from django.utils import timezone import requests +from maintenance_mode.core import (get_maintenance_mode, maintenance_mode_on, + set_maintenance_mode) + +from InvenTree.config import get_setting logger = logging.getLogger("inventree") @@ -506,3 +513,74 @@ def send_email(subject, body, recipients, from_email=None, html_message=None): fail_silently=False, html_message=html_message ) + + +@scheduled_task(ScheduledTask.DAILY) +def check_for_migrations(worker: bool = True): + """Checks if migrations are needed. + + If the setting auto_update is enabled we will start updateing. + """ + # Test if auto-updates are enabled + if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'): + return + + from plugin import registry + + plan = get_migration_plan() + + # Check if there are any open migrations + if not plan: + logger.info('There are no open migrations') + return + + logger.info('There are open migrations') + + # Log open migrations + for migration in plan: + logger.info(migration[0]) + + # Set the application to maintenance mode - no access from now on. + logger.info('Going into maintenance') + set_maintenance_mode(True) + logger.info('Mainentance mode is on now') + + # Check if we are worker - go kill all other workers then. + # Only the frontend workers run updates. + if worker: + logger.info('Current process is a worker - shutting down cluster') + + # Ok now we are ready to go ahead! + # To be sure we are in maintenance this is wrapped + with maintenance_mode_on(): + logger.info('Starting migrations') + print('Starting migrations') + + try: + call_command('migrate', interactive=False) + except NotSupportedError as e: # pragma: no cover + if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3': + raise e + logger.error(f'Error during migrations: {e}') + + print('Migrations done') + logger.info('Ran migrations') + + # Make sure we are out of maintenance again + logger.info('Checking InvenTree left maintenance mode') + if get_maintenance_mode(): + + logger.warning('Mainentance was still on - releasing now') + set_maintenance_mode(False) + logger.info('Released out of maintenance') + + # We should be current now - triggering full reload to make sure all models + # are loaded fully in their new state. + registry.reload_plugins(full_reload=True, force_reload=True) + + +def get_migration_plan(): + """Returns a list of migrations which are needed to be run.""" + executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) + plan = executor.migration_plan(executor.loader.graph.leaf_nodes()) + return plan diff --git a/InvenTree/InvenTree/test_tasks.py b/InvenTree/InvenTree/test_tasks.py index e2d0a6ce2c..38fd97a2d6 100644 --- a/InvenTree/InvenTree/test_tasks.py +++ b/InvenTree/InvenTree/test_tasks.py @@ -1,7 +1,11 @@ """Unit tests for task management.""" +import os from datetime import timedelta +from django.conf import settings +from django.core.management import call_command +from django.db.utils import NotSupportedError from django.test import TestCase from django.utils import timezone @@ -117,3 +121,32 @@ class InvenTreeTaskTests(TestCase): response = InvenTreeSetting.get_setting('_INVENTREE_LATEST_VERSION') self.assertNotEqual(response, '') self.assertTrue(bool(response)) + + def test_task_check_for_migrations(self): + """Test the task check_for_migrations.""" + # Update disabled + InvenTree.tasks.check_for_migrations() + + # Update enabled - no migrations + os.environ['INVENTREE_AUTO_UPDATE'] = 'True' + InvenTree.tasks.check_for_migrations() + + # Create migration + self.assertEqual(len(InvenTree.tasks.get_migration_plan()), 0) + call_command('makemigrations', ['InvenTree', '--empty'], interactive=False) + self.assertEqual(len(InvenTree.tasks.get_migration_plan()), 1) + + # Run with migrations - catch no foreigner error + try: + InvenTree.tasks.check_for_migrations() + except NotSupportedError as e: # pragma: no cover + if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3': + raise e + + # Cleanup + try: + migration_name = InvenTree.tasks.get_migration_plan()[0][0].name + '.py' + migration_path = settings.BASE_DIR / 'InvenTree' / 'migrations' / migration_name + migration_path.unlink() + except IndexError: # pragma: no cover + pass diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index 481d2935d6..f771930e0b 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -119,6 +119,10 @@ plugins_enabled: False #plugin_file: '/path/to/plugins.txt' #plugin_dir: '/path/to/plugins/' +# Set this variable to True to enable auto-migrations +# Alternatively, use the environment variable INVENTREE_AUTO_UPDATE +auto_update: False + # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation) # A list of strings representing the host/domain names that this Django site can serve. # Default behaviour is to allow all hosts (THIS IS NOT SECURE!) diff --git a/InvenTree/plugin/registry.py b/InvenTree/plugin/registry.py index 2132fd7ee6..b250a0bf19 100644 --- a/InvenTree/plugin/registry.py +++ b/InvenTree/plugin/registry.py @@ -162,8 +162,12 @@ class PluginsRegistry: logger.info('Finished loading plugins') - def unload_plugins(self): - """Unload and deactivate all IntegrationPlugins.""" + def unload_plugins(self, force_reload: bool = False): + """Unload and deactivate all IntegrationPlugins. + + Args: + force_reload (bool, optional): Also reload base apps. Defaults to False. + """ logger.info('Start unloading plugins') @@ -176,7 +180,7 @@ class PluginsRegistry: self._clean_registry() # deactivate all integrations - self._deactivate_plugins() + self._deactivate_plugins(force_reload=force_reload) # remove maintenance if not _maintenance: @@ -184,11 +188,12 @@ class PluginsRegistry: logger.info('Finished unloading plugins') - def reload_plugins(self, full_reload: bool = False): + def reload_plugins(self, full_reload: bool = False, force_reload: bool = False): """Safely reload. Args: full_reload (bool, optional): Reload everything - including plugin mechanism. Defaults to False. + force_reload (bool, optional): Also reload base apps. Defaults to False. """ # Do not reload whe currently loading if self.is_loading: @@ -197,8 +202,8 @@ class PluginsRegistry: logger.info('Start reloading plugins') with maintenance_mode_on(): - self.unload_plugins() - self.load_plugins(full_reload) + self.unload_plugins(force_reload=force_reload) + self.load_plugins(full_reload=full_reload) logger.info('Finished reloading plugins') @@ -470,9 +475,13 @@ class PluginsRegistry: self.activate_plugin_app(plugins, force_reload=force_reload, full_reload=full_reload) self.activate_plugin_url(plugins, force_reload=force_reload, full_reload=full_reload) - def _deactivate_plugins(self): - """Run deactivation functions for all plugins.""" - self.deactivate_plugin_app() + def _deactivate_plugins(self, force_reload: bool = False): + """Run deactivation functions for all plugins. + + Args: + force_reload (bool, optional): Also reload base apps. Defaults to False. + """ + self.deactivate_plugin_app(force_reload=force_reload) self.deactivate_plugin_schedule() self.deactivate_plugin_settings() # endregion @@ -655,8 +664,12 @@ class PluginsRegistry: plugin_path = plugin.__module__.split('.')[0] return plugin_path - def deactivate_plugin_app(self): - """Deactivate AppMixin plugins - some magic required.""" + def deactivate_plugin_app(self, force_reload: bool = False): + """Deactivate AppMixin plugins - some magic required. + + Args: + force_reload (bool, optional): Also reload base apps. Defaults to False. + """ # unregister models from admin for plugin_path in self.installed_apps: models = [] # the modelrefs need to be collected as poping an item in a iter is not welcomed @@ -693,7 +706,7 @@ class PluginsRegistry: # reset load flag and reload apps settings.INTEGRATION_APPS_LOADED = False - self._reload_apps() + self._reload_apps(force_reload=force_reload) # update urls to remove the apps from the site admin self._update_urls() diff --git a/InvenTree/plugin/serializers.py b/InvenTree/plugin/serializers.py index c7c892fe77..2f902e3d2a 100644 --- a/InvenTree/plugin/serializers.py +++ b/InvenTree/plugin/serializers.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from common.serializers import GenericReferencedSettingSerializer +from InvenTree.tasks import check_for_migrations, offload_task from plugin.models import NotificationUserSetting, PluginConfig, PluginSetting @@ -156,6 +157,9 @@ class PluginConfigInstallSerializer(serializers.Serializer): with open(settings.PLUGIN_FILE, "a") as plugin_file: plugin_file.write(f'{" ".join(install_name)} # Installed {timezone.now()} by {str(self.context["request"].user)}\n') + # Check for migrations + offload_task(check_for_migrations, worker=True) + return ret diff --git a/tasks.py b/tasks.py index 0bfff3f39d..3788b5f29b 100644 --- a/tasks.py +++ b/tasks.py @@ -620,4 +620,4 @@ def coverage(c): )) # Generate coverage report - c.run('coverage html') + c.run('coverage html -i')