Plugin auto migrate (#5668)

* Improved handling of race condition when saving setting value

* Improvements for managing pending migrations

- Inform user if there are outstanding migrations
- reload plugin registry (if necessary)

* Increase django-q polling time

According to this thread, should reduce multiple workers taking the same task:

https://github.com/Koed00/django-q/issues/183#issuecomment-239676084

* Revert default behavior

* Better logging

* Remove comment

* Update unit test

* Revert maintenance mode behaviour

* raise ValidationError in settings

InvenTree/InvenTree/apps.py

@@ -40,7 +40,6 @@ class InvenTreeConfig(AppConfig):
             return
 
         if canAppAccessDatabase() or settings.TESTING_ENV:
-            InvenTree.tasks.check_for_migrations(worker=False)
 
             self.remove_obsolete_tasks()
 
@@ -49,6 +48,8 @@ class InvenTreeConfig(AppConfig):
 
             if not isInTestMode():  # pragma: no cover
                 self.update_exchange_rates()
+                # Let the background worker check for migrations
+                InvenTree.tasks.offload_task(InvenTree.tasks.check_for_migrations)
 
         self.collect_notification_methods()
 

InvenTree/InvenTree/settings.py

@@ -751,6 +751,7 @@ Q_CLUSTER = {
     'orm': 'default',
     'cache': 'default',
     'sync': False,
+    'poll': 1.5,
 }
 
 # Configure django-q sentry integration

InvenTree/InvenTree/tasks.py

@@ -569,46 +569,60 @@ def run_backup():
     record_task_success('run_backup')
 
 
+def get_migration_plan():
+    """Returns a list of migrations which are needed to be run."""
+    executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
+    plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
+    return plan
+
+
 @scheduled_task(ScheduledTask.DAILY)
-def check_for_migrations(worker: bool = True):
+def check_for_migrations():
     """Checks if migrations are needed.
 
     If the setting auto_update is enabled we will start updating.
     """
-    # Test if auto-updates are enabled
-    if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'):
-        return
 
+    from common.models import InvenTreeSetting
     from plugin import registry
 
+    def set_pending_migrations(n: int):
+        """Helper function to inform the user about pending migrations"""
+
+        logger.info('There are %s pending migrations', n)
+        InvenTreeSetting.set_setting('_PENDING_MIGRATIONS', n, None)
+
+    logger.info("Checking for pending database migrations")
+
+    # Force plugin registry reload
+    registry.check_reload()
+
     plan = get_migration_plan()
 
+    n = len(plan)
+
     # Check if there are any open migrations
     if not plan:
-        logger.info('There are no open migrations')
+        set_pending_migrations(0)
         return
 
-    logger.info('There are open migrations')
+    set_pending_migrations(n)
+
+    # Test if auto-updates are enabled
+    if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'):
+        logger.info("Auto-update is disabled - skipping migrations")
+        return
 
     # Log open migrations
     for migration in plan:
-        logger.info(migration[0])
+        logger.info("- %s", str(migration[0]))
 
     # Set the application to maintenance mode - no access from now on.
-    logger.info('Going into maintenance')
     set_maintenance_mode(True)
-    logger.info('Mainentance mode is on now')
 
-    # Check if we are worker - go kill all other workers then.
-    # Only the frontend workers run updates.
-    if worker:
-        logger.info('Current process is a worker - shutting down cluster')
-
-    # Ok now we are ready to go ahead!
     # To be sure we are in maintenance this is wrapped
     with maintenance_mode_on():
-        logger.info('Starting migrations')
-        print('Starting migrations')
+        logger.info('Starting migration process...')
 
         try:
             call_command('migrate', interactive=False)
@@ -616,25 +630,17 @@ def check_for_migrations(worker: bool = True):
             if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
                 raise e
             logger.exception('Error during migrations: %s', e)
+        else:
+            set_pending_migrations(0)
 
-        print('Migrations done')
-        logger.info('Ran migrations')
+            logger.info("Completed %s migrations", n)
 
-    # Make sure we are out of maintenance again
-    logger.info('Checking InvenTree left maintenance mode')
+    # Make sure we are out of maintenance mode
     if get_maintenance_mode():
-
-        logger.warning('Mainentance was still on - releasing now')
+        logger.warning("Maintenance mode was not disabled - forcing it now")
         set_maintenance_mode(False)
-        logger.info('Released out of maintenance')
+        logger.info("Manually released maintenance mode")
 
     # We should be current now - triggering full reload to make sure all models
     # are loaded fully in their new state.
-    registry.reload_plugins(full_reload=True, force_reload=True)
-
-
-def get_migration_plan():
-    """Returns a list of migrations which are needed to be run."""
-    executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
-    plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
-    return plan
+    registry.reload_plugins(full_reload=True, force_reload=True, collect=True)

InvenTree/common/models.py

@@ -595,6 +595,8 @@ class BaseInvenTreeSetting(models.Model):
         if change_user is not None and not change_user.is_staff:
             return
 
+        attempts = int(kwargs.get('attempts', 3))
+
         filters = {
             'key__iexact': key,
 
@@ -615,8 +617,22 @@ class BaseInvenTreeSetting(models.Model):
         if setting.is_bool():
             value = InvenTree.helpers.str2bool(value)
 
-        setting.value = str(value)
-        setting.save()
+        try:
+            setting.value = str(value)
+            setting.save()
+        except ValidationError as exc:
+            # We need to know about validation errors
+            raise exc
+        except IntegrityError:
+            # Likely a race condition has caused a duplicate entry to be created
+            if attempts > 0:
+                # Try again
+                logger.info("Duplicate setting key '%s' for %s - trying again", key, str(cls))
+                cls.set_setting(key, value, change_user, create=create, attempts=attempts - 1, **kwargs)
+        except Exception as exc:
+            # Some other error
+            logger.exception("Error setting setting '%s' for %s: %s", key, str(cls), str(type(exc)))
+            pass
 
     key = models.CharField(max_length=50, blank=False, unique=False, help_text=_('Settings key (must be unique - case insensitive)'))
 
@@ -1050,6 +1066,13 @@ class InvenTreeSetting(BaseInvenTreeSetting):
             'hidden': True,
         },
 
+        '_PENDING_MIGRATIONS': {
+            'name': _('Pending migrations'),
+            'description': _('Number of pending database migrations'),
+            'default': 0,
+            'validator': int,
+        },
+
         'INVENTREE_INSTANCE': {
             'name': _('Server Instance Name'),
             'default': 'InvenTree',

InvenTree/common/tests.py

@@ -335,8 +335,10 @@ class GlobalSettingsApiTest(InvenTreeAPITestCase):
 
         response = self.get(url, expected_code=200)
 
+        n_public_settings = len([k for k in InvenTreeSetting.SETTINGS.keys() if not k.startswith('_')])
+
         # Number of results should match the number of settings
-        self.assertEqual(len(response.data), len(InvenTreeSetting.SETTINGS.keys()))
+        self.assertEqual(len(response.data), n_public_settings)
 
     def test_company_name(self):
         """Test a settings object lifecycle e2e."""

InvenTree/plugin/serializers.py

@@ -146,8 +146,15 @@ class PluginActivateSerializer(serializers.Serializer):
     def update(self, instance, validated_data):
         """Apply the new 'active' value to the plugin instance"""
 
+        from InvenTree.tasks import check_for_migrations, offload_task
+
         instance.active = validated_data.get('active', True)
         instance.save()
+
+        if instance.active:
+            # A plugin has just been activated - check for database migrations
+            offload_task(check_for_migrations)
+
         return instance
 
 

InvenTree/templates/base.html

@@ -8,6 +8,7 @@
 {% settings_value 'RETURNORDER_ENABLED' as return_order_enabled %}
 {% settings_value "REPORT_ENABLE" as report_enabled %}
 {% settings_value "SERVER_RESTART_REQUIRED" as server_restart_required %}
+{% settings_value "_PENDING_MIGRATIONS" as pending_migrations %}
 {% settings_value "LABEL_ENABLE" as labels_enabled %}
 {% inventree_show_about user as show_about %}
 
@@ -106,6 +107,16 @@
                     </small>
                 </div>
                 {% endif %}
+                {% if pending_migrations > 0 %}
+                <div id='alert-pending-migrations' class='alert alert-danger' role='alert'>
+                    <span class='fas fa-database'></span>
+                    <strong>{% trans "Pending Database Migrations" %}</strong>
+                    <small>
+                        <br>
+                        {% trans "There are pending database migrations which require attention" %}. {% trans "Contact your system administrator for further information" %}
+                    </small>
+                </div>
+                {% endif %}
             </div>
             {% endblock alerts %}
 

docker.dev.env

@@ -17,3 +17,6 @@ INVENTREE_DB_PASSWORD=pgpassword
 
 # Enable custom plugins?
 INVENTREE_PLUGINS_ENABLED=True
+
+# Auto run migrations?
+INVENTREE_AUTO_UPDATE=False

docker/production/.env

@@ -47,6 +47,9 @@ INVENTREE_GUNICORN_TIMEOUT=90
 # Enable custom plugins?
 INVENTREE_PLUGINS_ENABLED=False
 
+# Run migrations automatically?
+INVENTREE_AUTO_UPDATE=False
+
 # Image tag that should be used
 INVENTREE_TAG=stable