Add db an media backups (#3783)

* [FR] Add backup into inventrees lifecycle
Fixes #3778

* Add env to testing enviros

* block backup from running app commands

* Add more commands

* fix postgres version

* Update used env

* add daily task to run backups

* add installer changes

.devcontainer/devcontainer.json

@@ -71,6 +71,7 @@
     "INVENTREE_DB_NAME": "/workspaces/InvenTree/dev/database.sqlite3",
     "INVENTREE_MEDIA_ROOT": "/workspaces/InvenTree/dev/media",
     "INVENTREE_STATIC_ROOT": "/workspaces/InvenTree/dev/static",
+    "INVENTREE_BACKUP_DIR": "/workspaces/InvenTree/dev/backup",
     "INVENTREE_CONFIG_FILE": "/workspaces/InvenTree/dev/config.yaml",
     "INVENTREE_SECRET_KEY_FILE": "/workspaces/InvenTree/dev/secret_key.txt",
     "INVENTREE_PLUGIN_DIR": "/workspaces/InvenTree/dev/plugins",

.github/workflows/check_translations.yaml

@@ -20,6 +20,7 @@ jobs:
       INVENTREE_DEBUG: info
       INVENTREE_MEDIA_ROOT: ./media
       INVENTREE_STATIC_ROOT: ./static
+      INVENTREE_BACKUP_DIR: ./backup
 
     steps:
       - name: Checkout Code

.github/workflows/qc_checks.yaml

@@ -22,6 +22,7 @@ env:
   INVENTREE_DB_NAME: inventree
   INVENTREE_MEDIA_ROOT: ../test_inventree_media
   INVENTREE_STATIC_ROOT: ../test_inventree_static
+  INVENTREE_BACKUP_DIR: ../test_inventree_backup
 
 jobs:
   pep_style:
@@ -199,7 +200,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres
+        image: postgres:14
         env:
           POSTGRES_USER: inventree
           POSTGRES_PASSWORD: password

.github/workflows/translations.yml

@@ -17,6 +17,7 @@ jobs:
       INVENTREE_DEBUG: info
       INVENTREE_MEDIA_ROOT: ./media
       INVENTREE_STATIC_ROOT: ./static
+      INVENTREE_BACKUP_DIR: ./backup
 
     steps:
       - name: Checkout Code

.gitpod.yml

@@ -5,6 +5,7 @@ tasks:
       export INVENTREE_DB_NAME='/workspace/InvenTree/dev/database.sqlite3'
       export INVENTREE_MEDIA_ROOT='/workspace/InvenTree/inventree-data/media'
       export INVENTREE_STATIC_ROOT='/workspace/InvenTree/dev/static'
+      export INVENTREE_BACKUP_DIR='/workspace/InvenTree/dev/backup'
       export PIP_USER='no'
 
       sudo apt install -y gettext
@@ -24,6 +25,7 @@ tasks:
       export INVENTREE_DB_NAME='/workspace/InvenTree/dev/database.sqlite3'
       export INVENTREE_MEDIA_ROOT='/workspace/InvenTree/inventree-data/media'
       export INVENTREE_STATIC_ROOT='/workspace/InvenTree/dev/static'
+      export INVENTREE_BACKUP_DIR='/workspace/InvenTree/dev/backup'
 
       source venv/bin/activate
       inv server

.pkgr.yml

@@ -11,6 +11,7 @@ env:
   - INVENTREE_PLUGINS_ENABLED
   - INVENTREE_MEDIA_ROOT=/opt/inventree/media
   - INVENTREE_STATIC_ROOT=/opt/inventree/static
+  - INVENTREE_BACKUP_DIR=/opt/inventree/backup
   - INVENTREE_PLUGIN_FILE=/opt/inventree/plugins.txt
   - INVENTREE_CONFIG_FILE=/opt/inventree/config.yaml
 after_install: contrib/packager.io/postinstall.sh

Dockerfile

@@ -31,6 +31,7 @@ ENV INVENTREE_MNG_DIR="${INVENTREE_HOME}/InvenTree"
 ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data"
 ENV INVENTREE_STATIC_ROOT="${INVENTREE_DATA_DIR}/static"
 ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media"
+ENV INVENTREE_BACKUP_DIR="${INVENTREE_DATA_DIR}/backup"
 ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DATA_DIR}/plugins"
 
 # InvenTree configuration files

InvenTree/InvenTree/apps.py

@@ -117,6 +117,12 @@ class InvenTreeConfig(AppConfig):
             schedule_type=Schedule.DAILY
         )
 
+        # Make regular backups
+        InvenTree.tasks.schedule_task(
+            'InvenTree.tasks.run_backup',
+            schedule_type=Schedule.DAILY,
+        )
+
     def update_exchange_rates(self):  # pragma: no cover
         """Update exchange rates each time the server is started.
 

InvenTree/InvenTree/config.py

@@ -160,6 +160,22 @@ def get_static_dir(create=True):
     return sd
 
 
+def get_backup_dir(create=True):
+    """Return the absolute path for the backup directory"""
+
+    bd = get_setting('INVENTREE_BACKUP_DIR', 'backup_dir')
+
+    if not bd:
+        raise FileNotFoundError('INVENTREE_BACKUP_DIR not specified')
+
+    bd = Path(bd).resolve()
+
+    if create:
+        bd.mkdir(parents=True, exist_ok=True)
+
+    return bd
+
+
 def get_plugin_file():
     """Returns the path of the InvenTree plugins specification file.
 

InvenTree/InvenTree/ready.py

@@ -35,6 +35,12 @@ def canAppAccessDatabase(allow_test: bool = False, allow_plugins: bool = False):
         'collectstatic',
         'makemessages',
         'compilemessages',
+        'backup',
+        'dbbackup',
+        'mediabackup',
+        'restore',
+        'dbrestore',
+        'mediarestore',
     ]
 
     if not allow_test:

InvenTree/InvenTree/settings.py

@@ -131,6 +131,11 @@ STATIC_COLOR_THEMES_DIR = STATIC_ROOT.joinpath('css', 'color-themes').resolve()
 # Web URL endpoint for served media files
 MEDIA_URL = '/media/'
 
+# Backup directories
+DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage'
+DBBACKUP_STORAGE_OPTIONS = {'location': config.get_backup_dir()}
+DBBACKUP_SEND_EMAIL = False
+
 # Application definition
 
 INSTALLED_APPS = [
@@ -176,6 +181,7 @@ INSTALLED_APPS = [
     'error_report',                         # Error reporting in the admin interface
     'django_q',
     'formtools',                            # Form wizard tools
+    'dbbackup',                             # Backups - django-dbbackup
 
     'allauth',                              # Base app for SSO
     'allauth.account',                      # Extend user with accounts

InvenTree/InvenTree/tasks.py

@@ -9,6 +9,7 @@ from datetime import timedelta
 from django.conf import settings
 from django.core import mail as django_mail
 from django.core.exceptions import AppRegistryNotReady
+from django.core.management import call_command
 from django.db.utils import OperationalError, ProgrammingError
 from django.utils import timezone
 
@@ -272,6 +273,12 @@ def update_exchange_rates():
         logger.error(f"Error updating exchange rates: {e}")
 
 
+def run_backup():
+    """Run the backup command."""
+    call_command("dbbackup", noinput=True, clean=True, compress=True, interactive=False)
+    call_command("mediabackup", noinput=True, clean=True, compress=True, interactive=False)
+
+
 def send_email(subject, body, recipients, from_email=None, html_message=None):
     """Send an email with the specified subject and body, to the specified recipients list."""
     if type(recipients) == str:

InvenTree/config_template.yaml

@@ -142,6 +142,9 @@ cors:
 # STATIC_ROOT is the local filesystem location for storing static files
 #static_root: '/home/inventree/data/static'
 
+# BACKUP_DIR is the local filesystem location for storing backups
+#backup_dir: '/home/inventree/data/backup'
+
 # Background worker options
 background:
   workers: 4

contrib/packager.io/functions.sh

@@ -98,6 +98,7 @@ function detect_envs() {
     # Parse the config file
     export INVENTREE_MEDIA_ROOT=$conf | jq '.[].media_root'
     export INVENTREE_STATIC_ROOT=$conf | jq '.[].static_root'
+    export INVENTREE_BACKUP_DIR=$conf | jq '.[].backup_dir'
     export INVENTREE_PLUGINS_ENABLED=$conf | jq '.[].plugins_enabled'
     export INVENTREE_PLUGIN_FILE=$conf | jq '.[].plugin_file'
     export INVENTREE_SECRET_KEY_FILE=$conf | jq '.[].secret_key_file'
@@ -119,6 +120,7 @@ function detect_envs() {
 
     export INVENTREE_MEDIA_ROOT=${INVENTREE_MEDIA_ROOT:-${DATA_DIR}/media}
     export INVENTREE_STATIC_ROOT=${DATA_DIR}/static
+    export INVENTREE_BACKUP_DIR=${DATA_DIR}/backup
     export INVENTREE_PLUGINS_ENABLED=true
     export INVENTREE_PLUGIN_FILE=${CONF_DIR}/plugins.txt
     export INVENTREE_SECRET_KEY_FILE=${CONF_DIR}/secret_key.txt
@@ -137,6 +139,7 @@ function detect_envs() {
   echo "# Collected environment variables:"
   echo "#    INVENTREE_MEDIA_ROOT=${INVENTREE_MEDIA_ROOT}"
   echo "#    INVENTREE_STATIC_ROOT=${INVENTREE_STATIC_ROOT}"
+  echo "#    INVENTREE_BACKUP_DIR=${INVENTREE_BACKUP_DIR}"
   echo "#    INVENTREE_PLUGINS_ENABLED=${INVENTREE_PLUGINS_ENABLED}"
   echo "#    INVENTREE_PLUGIN_FILE=${INVENTREE_PLUGIN_FILE}"
   echo "#    INVENTREE_SECRET_KEY_FILE=${INVENTREE_SECRET_KEY_FILE}"
@@ -250,6 +253,8 @@ function set_env() {
   sed -i s=#media_root:\ \'/home/inventree/data/media\'=media_root:\ \'${INVENTREE_MEDIA_ROOT}\'=g ${INVENTREE_CONFIG_FILE}
   # Static Root
   sed -i s=#static_root:\ \'/home/inventree/data/static\'=static_root:\ \'${INVENTREE_STATIC_ROOT}\'=g ${INVENTREE_CONFIG_FILE}
+  # Backup dir
+  sed -i s=#backup_dir:\ \'/home/inventree/data/backup\'=backup_dir:\ \'${INVENTREE_BACKUP_DIR}\'=g ${INVENTREE_CONFIG_FILE}
   # Plugins enabled
   sed -i s=plugins_enabled:\ False=plugins_enabled:\ ${INVENTREE_PLUGINS_ENABLED}=g ${INVENTREE_CONFIG_FILE}
   # Plugin file

docker/init.sh

@@ -13,6 +13,11 @@ if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then
     mkdir -p $INVENTREE_MEDIA_ROOT
 fi
 
+if [[ ! -d "$INVENTREE_BACKUP_DIR" ]]; then
+    echo "Creating directory $INVENTREE_BACKUP_DIR"
+    mkdir -p $INVENTREE_BACKUP_DIR
+fi
+
 # Check if "config.yaml" has been copied into the correct location
 if test -f "$INVENTREE_CONFIG_FILE"; then
     echo "$INVENTREE_CONFIG_FILE exists - skipping"

requirements.in

@@ -7,6 +7,7 @@ django-allauth-2fa                      # MFA / 2FA
 django-cleanup                          # Automated deletion of old / unused uploaded files
 django-cors-headers                     # CORS headers extension for DRF
 django-crispy-forms                     # Form helpers
+django-dbbackup                         # Backup / restore of database and media files
 django-error-report                     # Error report viewer for the admin interface
 django-filter                           # Extended filtering options
 django-formtools                        # Form wizard tools

requirements.txt

@@ -48,6 +48,7 @@ django==3.2.16
     #   django-allauth
     #   django-allauth-2fa
     #   django-cors-headers
+    #   django-dbbackup
     #   django-error-report
     #   django-filter
     #   django-formtools
@@ -79,6 +80,8 @@ django-cors-headers==3.13.0
     # via -r requirements.in
 django-crispy-forms==1.14.0
     # via -r requirements.in
+django-dbbackup==4.0.2
+    # via -r requirements.in
 django-error-report==0.2.0
     # via -r requirements.in
 django-filter==22.1
@@ -181,6 +184,7 @@ pytz==2022.4
     # via
     #   babel
     #   django
+    #   django-dbbackup
     #   djangorestframework
 pyyaml==6.0
     # via tablib

tasks.py

@@ -196,7 +196,27 @@ def translate(c):
     manage(c, "compilemessages")
 
 
-@task(post=[rebuild_models, rebuild_thumbnails])
+@task
+def backup(c):
+    """Backup the database and media files."""
+
+    print("Backing up InvenTree database...")
+    manage(c, "dbbackup --noinput --clean --compress")
+    print("Backing up InvenTree media files...")
+    manage(c, "mediabackup --noinput --clean --compress")
+
+
+@task
+def restore(c):
+    """Restore the database and media files."""
+
+    print("Restoring InvenTree database...")
+    manage(c, "dbrestore --noinput --uncompress")
+    print("Restoring InvenTree media files...")
+    manage(c, "mediarestore --noinput --uncompress")
+
+
+@task(pre=[backup, ], post=[rebuild_models, rebuild_thumbnails])
 def migrate(c):
     """Performs database migrations.