Backup task fixes (#4307)

* Ensure 'retry' is always greater than timeout

* Adds setting for controlling how many days between automated backups

* Adds configuration option for max_attempts

* Update for daily backup task

- Prevent backup attempts from ocurring too frequently
- Add setting for controlling how many days between backups

* Exit early

InvenTree/InvenTree/settings.py

@@ -615,14 +615,16 @@ else:
         },
     }
 
+_q_worker_timeout = int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90))
+
 # django-q background worker configuration
 Q_CLUSTER = {
     'name': 'InvenTree',
     'label': 'Background Tasks',
     'workers': int(get_setting('INVENTREE_BACKGROUND_WORKERS', 'background.workers', 4)),
-    'timeout': int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)),
+    'timeout': _q_worker_timeout,
-    'retry': 120,
+    'retry': min(120, _q_worker_timeout + 30),
-    'max_attempts': 5,
+    'max_attempts': int(get_setting('INVENTREE_BACKGROUND_MAX_ATTEMPTS', 'background.max_attempts', 5)),
     'queue_limit': 50,
     'catch_up': False,
     'bulk': 10,

InvenTree/InvenTree/tasks.py

@@ -3,10 +3,12 @@
 import json
 import logging
 import os
+import random
 import re
+import time
 import warnings
 from dataclasses import dataclass
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import Callable, List
 
 from django.conf import settings
@@ -428,9 +430,66 @@ def run_backup():
     """Run the backup command."""
     from common.models import InvenTreeSetting
 
-    if InvenTreeSetting.get_setting('INVENTREE_BACKUP_ENABLE'):
+    if not InvenTreeSetting.get_setting('INVENTREE_BACKUP_ENABLE', False, cache=False):
-        call_command("dbbackup", noinput=True, clean=True, compress=True, interactive=False)
+        # Backups are not enabled - exit early
-        call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False)
+        return
+
+    logger.info("Performing automated database backup task")
+
+    # Sleep a random number of seconds to prevent worker conflict
+    time.sleep(random.randint(1, 5))
+
+    # Check for records of previous backup attempts
+    last_attempt = InvenTreeSetting.get_setting('INVENTREE_BACKUP_ATTEMPT', '', cache=False)
+    last_success = InvenTreeSetting.get_setting('INVENTREE_BACKUP_SUCCESS', '', cache=False)
+
+    try:
+        backup_n_days = int(InvenTreeSetting.get_setting('INVENTREE_BACKUP_DAYS', 1, cache=False))
+    except Exception:
+        backup_n_days = 1
+
+    if last_attempt:
+        try:
+            last_attempt = datetime.fromisoformat(last_attempt)
+        except ValueError:
+            last_attempt = None
+
+    if last_attempt:
+        # Do not attempt if the 'last attempt' at backup was within 12 hours
+        threshold = timezone.now() - timezone.timedelta(hours=12)
+
+        if last_attempt > threshold:
+            logger.info('Last backup attempt was too recent - skipping backup operation')
+            return
+
+    # Record the timestamp of most recent backup attempt
+    InvenTreeSetting.set_setting('INVENTREE_BACKUP_ATTEMPT', timezone.now().isoformat(), None)
+
+    if not last_attempt:
+        # If there is no record of a previous attempt, exit quickly
+        # This prevents the backup operation from happening when the server first launches, for example
+        logger.info("No previous backup attempts recorded - waiting until tomorrow")
+        return
+
+    if last_success:
+        try:
+            last_success = datetime.fromisoformat(last_success)
+        except ValueError:
+            last_success = None
+
+    # Exit early if the backup was successful within the number of required days
+    if last_success:
+        threshold = timezone.now() - timezone.timedelta(days=backup_n_days)
+
+        if last_success > threshold:
+            logger.info('Last successful backup was too recent - skipping backup operation')
+            return
+
+    call_command("dbbackup", noinput=True, clean=True, compress=True, interactive=False)
+    call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False)
+
+    # Record the timestamp of most recent backup success
+    InvenTreeSetting.set_setting('INVENTREE_BACKUP_SUCCESS', datetime.now().isoformat(), None)
 
 
 def send_email(subject, body, recipients, from_email=None, html_message=None):

InvenTree/common/models.py

@@ -969,6 +969,16 @@ class InvenTreeSetting(BaseInvenTreeSetting):
             'default': False,
         },
 
+        'INVENTREE_BACKUP_DAYS': {
+            'name': _('Days Between Backup'),
+            'description': _('Specify number of days between automated backup events'),
+            'validator': [
+                int,
+                MinValueValidator(1),
+            ],
+            'default': 1,
+        },
+
         'INVENTREE_DELETE_TASKS_DAYS': {
             'name': _('Delete Old Tasks'),
             'description': _('Background task results will be deleted after specified number of days'),

InvenTree/config_template.yaml

@@ -152,6 +152,7 @@ backup_storage: django.core.files.storage.FileSystemStorage
 background:
   workers: 4
   timeout: 90
+  max_attempts: 5
 
 # Optional URL schemes to allow in URL fields
 # By default, only the following schemes are allowed: ['http', 'https', 'ftp', 'ftps']

InvenTree/templates/InvenTree/settings/global.html

@@ -25,6 +25,7 @@
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_REQUIRE_CONFIRM" icon="fa-check" %}
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_TREE_DEPTH" icon="fa-sitemap" %}
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_BACKUP_ENABLE" icon="fa-hdd" %}
+        {% include "InvenTree/settings/setting.html" with key="INVENTREE_BACKUP_DAYS" icon="fa-calendar-alt" %}
         <tr><td colspan='5'></td></tr>
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_DELETE_TASKS_DAYS" icon="fa-calendar-alt" %}
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_DELETE_ERRORS_DAYS" icon="fa-calendar-alt" %}