From 9e6466b910c2fce27ce96cd976727447a833d405 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 6 Feb 2023 19:58:18 +1100 Subject: [PATCH] Backup task fixes (#4307) * Ensure 'retry' is always greater than timeout * Adds setting for controlling how many days between automated backups * Adds configuration option for max_attempts * Update for daily backup task - Prevent backup attempts from ocurring too frequently - Add setting for controlling how many days between backups * Exit early --- InvenTree/InvenTree/settings.py | 8 ++- InvenTree/InvenTree/tasks.py | 67 +++++++++++++++++-- InvenTree/common/models.py | 10 +++ InvenTree/config_template.yaml | 1 + .../templates/InvenTree/settings/global.html | 1 + 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index d790d65631..e773a1d378 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -615,14 +615,16 @@ else: }, } +_q_worker_timeout = int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)) + # django-q background worker configuration Q_CLUSTER = { 'name': 'InvenTree', 'label': 'Background Tasks', 'workers': int(get_setting('INVENTREE_BACKGROUND_WORKERS', 'background.workers', 4)), - 'timeout': int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)), - 'retry': 120, - 'max_attempts': 5, + 'timeout': _q_worker_timeout, + 'retry': min(120, _q_worker_timeout + 30), + 'max_attempts': int(get_setting('INVENTREE_BACKGROUND_MAX_ATTEMPTS', 'background.max_attempts', 5)), 'queue_limit': 50, 'catch_up': False, 'bulk': 10, diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 32c88e42a4..753fc44a65 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -3,10 +3,12 @@ import json import logging import os +import random import re +import time import warnings from dataclasses import dataclass -from datetime import timedelta +from datetime import datetime, timedelta from typing import Callable, List from django.conf import settings @@ -428,9 +430,66 @@ def run_backup(): """Run the backup command.""" from common.models import InvenTreeSetting - if InvenTreeSetting.get_setting('INVENTREE_BACKUP_ENABLE'): - call_command("dbbackup", noinput=True, clean=True, compress=True, interactive=False) - call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False) + if not InvenTreeSetting.get_setting('INVENTREE_BACKUP_ENABLE', False, cache=False): + # Backups are not enabled - exit early + return + + logger.info("Performing automated database backup task") + + # Sleep a random number of seconds to prevent worker conflict + time.sleep(random.randint(1, 5)) + + # Check for records of previous backup attempts + last_attempt = InvenTreeSetting.get_setting('INVENTREE_BACKUP_ATTEMPT', '', cache=False) + last_success = InvenTreeSetting.get_setting('INVENTREE_BACKUP_SUCCESS', '', cache=False) + + try: + backup_n_days = int(InvenTreeSetting.get_setting('INVENTREE_BACKUP_DAYS', 1, cache=False)) + except Exception: + backup_n_days = 1 + + if last_attempt: + try: + last_attempt = datetime.fromisoformat(last_attempt) + except ValueError: + last_attempt = None + + if last_attempt: + # Do not attempt if the 'last attempt' at backup was within 12 hours + threshold = timezone.now() - timezone.timedelta(hours=12) + + if last_attempt > threshold: + logger.info('Last backup attempt was too recent - skipping backup operation') + return + + # Record the timestamp of most recent backup attempt + InvenTreeSetting.set_setting('INVENTREE_BACKUP_ATTEMPT', timezone.now().isoformat(), None) + + if not last_attempt: + # If there is no record of a previous attempt, exit quickly + # This prevents the backup operation from happening when the server first launches, for example + logger.info("No previous backup attempts recorded - waiting until tomorrow") + return + + if last_success: + try: + last_success = datetime.fromisoformat(last_success) + except ValueError: + last_success = None + + # Exit early if the backup was successful within the number of required days + if last_success: + threshold = timezone.now() - timezone.timedelta(days=backup_n_days) + + if last_success > threshold: + logger.info('Last successful backup was too recent - skipping backup operation') + return + + call_command("dbbackup", noinput=True, clean=True, compress=True, interactive=False) + call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False) + + # Record the timestamp of most recent backup success + InvenTreeSetting.set_setting('INVENTREE_BACKUP_SUCCESS', datetime.now().isoformat(), None) def send_email(subject, body, recipients, from_email=None, html_message=None): diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 9297430556..f42b9ee5c5 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -969,6 +969,16 @@ class InvenTreeSetting(BaseInvenTreeSetting): 'default': False, }, + 'INVENTREE_BACKUP_DAYS': { + 'name': _('Days Between Backup'), + 'description': _('Specify number of days between automated backup events'), + 'validator': [ + int, + MinValueValidator(1), + ], + 'default': 1, + }, + 'INVENTREE_DELETE_TASKS_DAYS': { 'name': _('Delete Old Tasks'), 'description': _('Background task results will be deleted after specified number of days'), diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index b8c15fc413..6a0fbc37f5 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -152,6 +152,7 @@ backup_storage: django.core.files.storage.FileSystemStorage background: workers: 4 timeout: 90 + max_attempts: 5 # Optional URL schemes to allow in URL fields # By default, only the following schemes are allowed: ['http', 'https', 'ftp', 'ftps'] diff --git a/InvenTree/templates/InvenTree/settings/global.html b/InvenTree/templates/InvenTree/settings/global.html index b7516e641d..ba71fed465 100644 --- a/InvenTree/templates/InvenTree/settings/global.html +++ b/InvenTree/templates/InvenTree/settings/global.html @@ -25,6 +25,7 @@ {% include "InvenTree/settings/setting.html" with key="INVENTREE_REQUIRE_CONFIRM" icon="fa-check" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_TREE_DEPTH" icon="fa-sitemap" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_BACKUP_ENABLE" icon="fa-hdd" %} + {% include "InvenTree/settings/setting.html" with key="INVENTREE_BACKUP_DAYS" icon="fa-calendar-alt" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_DELETE_TASKS_DAYS" icon="fa-calendar-alt" %} {% include "InvenTree/settings/setting.html" with key="INVENTREE_DELETE_ERRORS_DAYS" icon="fa-calendar-alt" %}