Update demo data hook to copy media files (#3441)

* Simplify settings.py / config.py - get_setting function has been streamlined - move some functions into config.py * Spelling fix: IGNORRED is IGNORED * Ensure yaml is installed as part of docker image - invoke path is still mucking us around * Fix broken migration * Copy media files from demo dataset when installing test data * Cleanup settings.py * Fix for configuration file traversal * Line fix * Update quickstart guide for docker * Allow plugin file and plugin dir to be specified in configuration file * Cleanup config template file * Allow secret_key information to be provided in configuration file * Adjust root paths for CI tests * resolve paths * Revert paths for CI step * remove dead code * Revert configuration variables to old names - Prevent breaking changes * Simplify secret key generation * Fix default timeout for background worker process * Revert change for customization options
2024-08-30 18:33:04 +00:00 · 2022-07-31 23:16:58 +10:00 · 2022-07-31 23:16:58 +10:00 · e9b0f02ecd
commit e9b0f02ecd
parent d4eae9da11
13 changed files with 311 additions and 343 deletions
--- a/.gitignore
+++ b/.gitignore
@ -50,7 +50,6 @@ docs/_build
 inventree_media
 inventree_static
 static_i18n
-inventree-data

 # Local config file
 config.yaml
@ -79,6 +78,8 @@ js_tmp/

 # Development files
 dev/
+data/
+env/

 # Locale stats file
 locale_stats.json
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -18,6 +18,7 @@ pip install invoke && invoke setup-dev --tests

 ```bash
 git clone https://github.com/inventree/InvenTree.git && cd InvenTree
+docker compose run inventree-dev-server invoke install
 docker compose run inventree-dev-server invoke setup-test
 docker compose up -d
 ```
--- a/InvenTree/InvenTree/apps.py
+++ b/InvenTree/InvenTree/apps.py
@ -172,21 +172,13 @@ class InvenTreeConfig(AppConfig):
            return

        # get values
-        add_user = get_setting(
-            'INVENTREE_ADMIN_USER',
-            settings.CONFIG.get('admin_user', False)
-        )
-        add_email = get_setting(
-            'INVENTREE_ADMIN_EMAIL',
-            settings.CONFIG.get('admin_email', False)
-        )
-        add_password = get_setting(
-            'INVENTREE_ADMIN_PASSWORD',
-            settings.CONFIG.get('admin_password', False)
-        )
+        add_user = get_setting('INVENTREE_ADMIN_USER', 'admin_user')
+        add_email = get_setting('INVENTREE_ADMIN_EMAIL', 'admin_email')
+        add_password = get_setting('INVENTREE_ADMIN_PASSWORD', 'admin_password')

        # check if all values are present
        set_variables = 0
+
        for tested_var in [add_user, add_email, add_password]:
            if tested_var:
                set_variables += 1
--- a/InvenTree/InvenTree/config.py
+++ b/InvenTree/InvenTree/config.py
@ -2,18 +2,27 @@

 import logging
 import os
+import random
 import shutil
+import string
 from pathlib import Path

+import yaml
+
 logger = logging.getLogger('inventree')


+def is_true(x):
+    """Shortcut function to determine if a value "looks" like a boolean"""
+    return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true', 'on']
+
+
 def get_base_dir() -> Path:
    """Returns the base (top-level) InvenTree directory."""
    return Path(__file__).parent.parent.resolve()


-def get_config_file() -> Path:
+def get_config_file(create=True) -> Path:
    """Returns the path of the InvenTree configuration file.

    Note: It will be created it if does not already exist!
@ -28,7 +37,7 @@ def get_config_file() -> Path:
        # Config file is *not* specified - use the default
        cfg_filename = base_dir.joinpath('config.yaml').resolve()

-    if not cfg_filename.exists():
+    if not cfg_filename.exists() and create:
        print("InvenTree configuration file 'config.yaml' not found - creating default file")

        cfg_template = base_dir.joinpath("config_template.yaml")
@ -38,45 +47,159 @@ def get_config_file() -> Path:
    return cfg_filename


-def get_plugin_file():
-    """Returns the path of the InvenTree plugins specification file.
+def load_config_data() -> map:
+    """Load configuration data from the config file."""

-    Note: It will be created if it does not already exist!
-    """
-    # Check if the plugin.txt file (specifying required plugins) is specified
-    PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE')
+    cfg_file = get_config_file()

-    if not PLUGIN_FILE:
-        # If not specified, look in the same directory as the configuration file
-        config_dir = get_config_file().parent
-        PLUGIN_FILE = config_dir.joinpath('plugins.txt')
-    else:
-        # Make sure we are using a modern Path object
-        PLUGIN_FILE = Path(PLUGIN_FILE)
+    with open(cfg_file, 'r') as cfg:
+        data = yaml.safe_load(cfg)

-    if not PLUGIN_FILE.exists():
-        logger.warning("Plugin configuration file does not exist")
-        logger.info(f"Creating plugin file at '{PLUGIN_FILE}'")
-
-        # If opening the file fails (no write permission, for example), then this will throw an error
-        PLUGIN_FILE.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n")
-
-    return PLUGIN_FILE
+    return data


-def get_setting(environment_var, backup_val, default_value=None):
+def get_setting(env_var=None, config_key=None, default_value=None):
    """Helper function for retrieving a configuration setting value.

    - First preference is to look for the environment variable
    - Second preference is to look for the value of the settings file
    - Third preference is the default value
+
+    Arguments:
+        env_var: Name of the environment variable e.g. 'INVENTREE_STATIC_ROOT'
+        config_key: Key to lookup in the configuration file
+        default_value: Value to return if first two options are not provided
+
    """
-    val = os.getenv(environment_var)

-    if val is not None:
-        return val
+    # First, try to load from the environment variables
+    if env_var is not None:
+        val = os.getenv(env_var, None)

-    if backup_val is not None:
-        return backup_val
+        if val is not None:
+            return val

+    # Next, try to load from configuration file
+    if config_key is not None:
+        cfg_data = load_config_data()
+
+        result = None
+
+        # Hack to allow 'path traversal' in configuration file
+        for key in config_key.strip().split('.'):
+
+            if type(cfg_data) is not dict or key not in cfg_data:
+                result = None
+                break
+
+            result = cfg_data[key]
+            cfg_data = cfg_data[key]
+
+        if result is not None:
+            return result
+
+    # Finally, return the default value
    return default_value
+
+
+def get_boolean_setting(env_var=None, config_key=None, default_value=False):
+    """Helper function for retreiving a boolean configuration setting"""
+
+    return is_true(get_setting(env_var, config_key, default_value))
+
+
+def get_media_dir(create=True):
+    """Return the absolute path for the 'media' directory (where uploaded files are stored)"""
+
+    md = get_setting('INVENTREE_MEDIA_ROOT', 'media_root')
+
+    if not md:
+        raise FileNotFoundError('INVENTREE_MEDIA_ROOT not specified')
+
+    md = Path(md).resolve()
+
+    if create:
+        md.mkdir(parents=True, exist_ok=True)
+
+    return md
+
+
+def get_static_dir(create=True):
+    """Return the absolute path for the 'static' directory (where static files are stored)"""
+
+    sd = get_setting('INVENTREE_STATIC_ROOT', 'static_root')
+
+    if not sd:
+        raise FileNotFoundError('INVENTREE_STATIC_ROOT not specified')
+
+    sd = Path(sd).resolve()
+
+    if create:
+        sd.mkdir(parents=True, exist_ok=True)
+
+    return sd
+
+
+def get_plugin_file():
+    """Returns the path of the InvenTree plugins specification file.
+
+    Note: It will be created if it does not already exist!
+    """
+
+    # Check if the plugin.txt file (specifying required plugins) is specified
+    plugin_file = get_setting('INVENTREE_PLUGIN_FILE', 'plugin_file')
+
+    if not plugin_file:
+        # If not specified, look in the same directory as the configuration file
+        config_dir = get_config_file().parent
+        plugin_file = config_dir.joinpath('plugins.txt')
+    else:
+        # Make sure we are using a modern Path object
+        plugin_file = Path(plugin_file)
+
+    if not plugin_file.exists():
+        logger.warning("Plugin configuration file does not exist - creating default file")
+        logger.info(f"Creating plugin file at '{plugin_file}'")
+
+        # If opening the file fails (no write permission, for example), then this will throw an error
+        plugin_file.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n")
+
+    return plugin_file
+
+
+def get_secret_key():
+    """Return the secret key value which will be used by django.
+
+    Following options are tested, in descending order of preference:
+
+    A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data
+    B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
+    C) Look for default key file "secret_key.txt"
+    D) Create "secret_key.txt" if it does not exist
+    """
+
+    # Look for environment variable
+    if secret_key := get_setting('INVENTREE_SECRET_KEY', 'secret_key'):
+        logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY")  # pragma: no cover
+        return secret_key
+
+    # Look for secret key file
+    if secret_key_file := get_setting('INVENTREE_SECRET_KEY_FILE', 'secret_key_file'):
+        secret_key_file = Path(secret_key_file).resolve()
+    else:
+        # Default location for secret key file
+        secret_key_file = get_base_dir().joinpath("secret_key.txt").resolve()
+
+    if not secret_key_file.exists():
+        logger.info(f"Generating random key file at '{secret_key_file}'")
+
+        # Create a random key file
+        options = string.digits + string.ascii_letters + string.punctuation
+        key = ''.join([random.choice(options) for i in range(100)])
+        secret_key_file.write_text(key)
+
+    logger.info(f"Loading SECRET_KEY from '{secret_key_file}'")
+
+    key_data = secret_key_file.read_text().strip()
+
+    return key_data
--- a/InvenTree/InvenTree/exceptions.py
+++ b/InvenTree/InvenTree/exceptions.py
@ -29,7 +29,7 @@ def log_error(path):
    kind, info, data = sys.exc_info()

    # Check if the eror is on the ignore list
-    if kind in settings.IGNORRED_ERRORS:
+    if kind in settings.IGNORED_ERRORS:
        return

    Error.objects.create(
--- a/InvenTree/InvenTree/middleware.py
+++ b/InvenTree/InvenTree/middleware.py
@ -157,7 +157,7 @@ class InvenTreeExceptionProcessor(ExceptionProcessor):
        kind, info, data = sys.exc_info()

        # Check if the eror is on the ignore list
-        if kind in settings.IGNORRED_ERRORS:
+        if kind in settings.IGNORED_ERRORS:
            return

        return super().process_exception(request, exception)
--- a/InvenTree/InvenTree/settings.py
+++ b/InvenTree/InvenTree/settings.py
@ -11,9 +11,7 @@ database setup in this file.

 import logging
 import os
-import random
 import socket
-import string
 import sys
 from pathlib import Path

@ -24,22 +22,14 @@ from django.utils.translation import gettext_lazy as _

 import moneyed
 import sentry_sdk
-import yaml
 from sentry_sdk.integrations.django import DjangoIntegration

-from .config import get_base_dir, get_config_file, get_plugin_file, get_setting
-
-
-def _is_true(x):
-    # Shortcut function to determine if a value "looks" like a boolean
-    return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true']
-
-
-# Default Sentry DSN (can be overriden if user wants custom sentry integration)
-INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600'
+from . import config
+from .config import get_boolean_setting, get_setting

 # Determine if we are running in "test" mode e.g. "manage.py test"
 TESTING = 'test' in sys.argv
+
 # Are enviroment variables manipulated by tests? Needs to be set by testing code
 TESTING_ENV = False

@ -47,33 +37,17 @@ TESTING_ENV = False
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'

 # Build paths inside the project like this: BASE_DIR.joinpath(...)
-BASE_DIR = get_base_dir()
+BASE_DIR = config.get_base_dir()

-cfg_filename = get_config_file()
-
-with open(cfg_filename, 'r') as cfg:
-    CONFIG = yaml.safe_load(cfg)
-
-# We will place any config files in the same directory as the config file
-config_dir = cfg_filename.parent
+# Load configuration data
+CONFIG = config.load_config_data()

 # Default action is to run the system in Debug mode
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = _is_true(get_setting(
-    'INVENTREE_DEBUG',
-    CONFIG.get('debug', True)
-))
-
-DOCKER = _is_true(get_setting(
-    'INVENTREE_DOCKER',
-    False
-))
+DEBUG = get_boolean_setting('INVENTREE_DEBUG', 'debug', True)

 # Configure logging settings
-log_level = get_setting(
-    'INVENTREE_LOG_LEVEL',
-    CONFIG.get('log_level', 'WARNING')
-)
+log_level = get_setting('INVENTREE_LOG_LEVEL', 'log_level', 'WARNING')

 logging.basicConfig(
    level=log_level,
@ -105,78 +79,20 @@ LOGGING = {
 # Get a logger instance for this setup file
 logger = logging.getLogger("inventree")

-"""
-Specify a secret key to be used by django.
-
-Following options are tested, in descending order of preference:
-
-A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data
-B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
-C) Look for default key file "secret_key.txt"
-d) Create "secret_key.txt" if it does not exist
-"""
-
-if secret_key := os.getenv("INVENTREE_SECRET_KEY"):
-    # Secret key passed in directly
-    SECRET_KEY = secret_key.strip()  # pragma: no cover
-    logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY")  # pragma: no cover
-else:
-    # Secret key passed in by file location
-    key_file = os.getenv("INVENTREE_SECRET_KEY_FILE")
-
-    if key_file:
-        key_file = Path(key_file).resolve()  # pragma: no cover
-    else:
-        # default secret key location
-        key_file = BASE_DIR.joinpath("secret_key.txt").resolve()
-
-    if not key_file.exists():  # pragma: no cover
-        logger.info(f"Generating random key file at '{key_file}'")
-        # Create a random key file
-        options = string.digits + string.ascii_letters + string.punctuation
-        key = ''.join([random.choice(options) for i in range(100)])
-        key_file.write_text(key)
-
-    logger.info(f"Loading SECRET_KEY from '{key_file}'")
-
-    try:
-        SECRET_KEY = open(key_file, "r").read().strip()
-    except Exception:  # pragma: no cover
-        logger.exception(f"Couldn't load keyfile {key_file}")
-        sys.exit(-1)
+# Load SECRET_KEY
+SECRET_KEY = config.get_secret_key()

 # The filesystem location for served static files
-STATIC_ROOT = Path(
-    get_setting(
-        'INVENTREE_STATIC_ROOT',
-        CONFIG.get('static_root', None)
-    )
-).resolve()
+STATIC_ROOT = config.get_static_dir()

-if STATIC_ROOT is None:  # pragma: no cover
-    print("ERROR: INVENTREE_STATIC_ROOT directory not defined")
-    sys.exit(1)
-else:
-    # Ensure the root really is availalble
-    STATIC_ROOT.mkdir(parents=True, exist_ok=True)
-
-# The filesystem location for served static files
-MEDIA_ROOT = Path(
-    get_setting(
-        'INVENTREE_MEDIA_ROOT',
-        CONFIG.get('media_root', None)
-    )
-).resolve()
-
-if MEDIA_ROOT is None:  # pragma: no cover
-    print("ERROR: INVENTREE_MEDIA_ROOT directory is not defined")
-    sys.exit(1)
-else:
-    # Ensure the root really is availalble
-    MEDIA_ROOT.mkdir(parents=True, exist_ok=True)
+# The filesystem location for uploaded meadia files
+MEDIA_ROOT = config.get_media_dir()

 # List of allowed hosts (default = allow all)
-ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*'])
+ALLOWED_HOSTS = get_setting(
+    config_key='allowed_hosts',
+    default_value=['*']
+)

 # Cross Origin Resource Sharing (CORS) options

@ -184,13 +100,15 @@ ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*'])
 CORS_URLS_REGEX = r'^/api/.*$'

 # Extract CORS options from configuration file
-cors_opt = CONFIG.get('cors', None)
+CORS_ORIGIN_ALLOW_ALL = get_boolean_setting(
+    config_key='cors.allow_all',
+    default_value=False,
+)

-if cors_opt:
-    CORS_ORIGIN_ALLOW_ALL = cors_opt.get('allow_all', False)
-
-    if not CORS_ORIGIN_ALLOW_ALL:
-        CORS_ORIGIN_WHITELIST = cors_opt.get('whitelist', [])  # pragma: no cover
+CORS_ORIGIN_WHITELIST = get_setting(
+    config_key='cors.whitelist',
+    default_value=[]
+)

 # Web URL endpoint for served static files
 STATIC_URL = '/static/'
@ -214,12 +132,6 @@ STATIC_COLOR_THEMES_DIR = STATIC_ROOT.joinpath('css', 'color-themes')
 # Web URL endpoint for served media files
 MEDIA_URL = '/media/'

-if DEBUG:
-    logger.info("InvenTree running with DEBUG enabled")
-
-logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'")
-logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'")
-
 # Application definition

 INSTALLED_APPS = [
@ -320,6 +232,9 @@ INTERNAL_IPS = [
    '127.0.0.1',
 ]

+# Internal flag to determine if we are running in docker mode
+DOCKER = get_boolean_setting('INVENTREE_DOCKER', default_value=False)
+
 if DOCKER:  # pragma: no cover
    # Internal IP addresses are different when running under docker
    hostname, ___, ips = socket.gethostbyname_ex(socket.gethostname())
@ -334,7 +249,8 @@ if DEBUG:
 # Base URL for admin pages (default="admin")
 INVENTREE_ADMIN_URL = get_setting(
    'INVENTREE_ADMIN_URL',
-    CONFIG.get('admin_url', 'admin'),
+    config_key='admin_url',
+    default_value='admin'
 )

 ROOT_URLCONF = 'InvenTree.urls'
@ -498,7 +414,7 @@ if "postgres" in db_engine:  # pragma: no cover
        # long to connect to the database server
        # # seconds, 2 is minium allowed by libpq
        db_options["connect_timeout"] = int(
-            os.getenv("INVENTREE_DB_TIMEOUT", 2)
+            get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 2)
        )

    # Setup TCP keepalive
@ -509,23 +425,27 @@ if "postgres" in db_engine:  # pragma: no cover
    # # 0 - TCP Keepalives disabled; 1 - enabled
    if "keepalives" not in db_options:
        db_options["keepalives"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES", "1")
+            get_setting('INVENTREE_DB_TCP_KEEPALIVES', 'database.tcp_keepalives', 1)
        )
-    # # Seconds after connection is idle to send keep alive
+
+    # Seconds after connection is idle to send keep alive
    if "keepalives_idle" not in db_options:
        db_options["keepalives_idle"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_IDLE", "1")
+            get_setting('INVENTREE_DB_TCP_KEEPALIVES_IDLE', 'database.tcp_keepalives_idle', 1)
        )
-    # # Seconds after missing ACK to send another keep alive
+
+    # Seconds after missing ACK to send another keep alive
    if "keepalives_interval" not in db_options:
        db_options["keepalives_interval"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "1")
+            get_setting("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "database.tcp_keepalives_internal", "1")
        )
-    # # Number of missing ACKs before we close the connection
+
+    # Number of missing ACKs before we close the connection
    if "keepalives_count" not in db_options:
        db_options["keepalives_count"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "5")
+            get_setting("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "database.tcp_keepalives_count", "5")
        )
+
    # # Milliseconds for how long pending data should remain unacked
    # by the remote server
    # TODO: Supported starting in PSQL 11
@ -538,17 +458,11 @@ if "postgres" in db_engine:  # pragma: no cover
    # https://www.postgresql.org/docs/devel/transaction-iso.html
    # https://docs.djangoproject.com/en/3.2/ref/databases/#isolation-level
    if "isolation_level" not in db_options:
-        serializable = _is_true(
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
-        )
-        db_options["isolation_level"] = (
-            ISOLATION_LEVEL_SERIALIZABLE
-            if serializable
-            else ISOLATION_LEVEL_READ_COMMITTED
-        )
+        serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False)
+        db_options["isolation_level"] = ISOLATION_LEVEL_SERIALIZABLE if serializable else ISOLATION_LEVEL_READ_COMMITTED

 # Specific options for MySql / MariaDB backend
-if "mysql" in db_engine:  # pragma: no cover
+elif "mysql" in db_engine:  # pragma: no cover
    # TODO TCP time outs and keepalives

    # MariaDB's default isolation level is Repeatable Read which is
@ -558,15 +472,11 @@ if "mysql" in db_engine:  # pragma: no cover
    # https://mariadb.com/kb/en/mariadb-transactions-and-isolation-levels-for-sql-server-users/#changing-the-isolation-level
    # https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
    if "isolation_level" not in db_options:
-        serializable = _is_true(
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
-        )
-        db_options["isolation_level"] = (
-            "serializable" if serializable else "read committed"
-        )
+        serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False)
+        db_options["isolation_level"] = "serializable" if serializable else "read committed"

 # Specific options for sqlite backend
-if "sqlite" in db_engine:
+elif "sqlite" in db_engine:
    # TODO: Verify timeouts are not an issue because no network is involved for SQLite

    # SQLite's default isolation level is Serializable due to SQLite's
@ -591,13 +501,11 @@ DATABASES = {
    'default': db_config
 }

-_cache_config = CONFIG.get("cache", {})
-_cache_host = _cache_config.get("host", os.getenv("INVENTREE_CACHE_HOST"))
-_cache_port = _cache_config.get(
-    "port", os.getenv("INVENTREE_CACHE_PORT", "6379")
-)
+# Cache configuration
+cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None)
+cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379')

-if _cache_host:  # pragma: no cover
+if cache_host:  # pragma: no cover
    # We are going to rely upon a possibly non-localhost for our cache,
    # so don't wait too long for the cache as nothing in the cache should be
    # irreplacable.
@ -606,7 +514,7 @@ if _cache_host:  # pragma: no cover
        "SOCKET_CONNECT_TIMEOUT": int(os.getenv("CACHE_CONNECT_TIMEOUT", "2")),
        "SOCKET_TIMEOUT": int(os.getenv("CACHE_SOCKET_TIMEOUT", "2")),
        "CONNECTION_POOL_KWARGS": {
-            "socket_keepalive": _is_true(
+            "socket_keepalive": config.is_true(
                os.getenv("CACHE_TCP_KEEPALIVE", "1")
            ),
            "socket_keepalive_options": {
@ -628,7 +536,7 @@ if _cache_host:  # pragma: no cover
    CACHES = {
        "default": {
            "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": f"redis://{_cache_host}:{_cache_port}/0",
+            "LOCATION": f"redis://{cache_host}:{cache_port}/0",
            "OPTIONS": _cache_options,
        },
    }
@ -639,17 +547,11 @@ else:
        },
    }

-try:
-    # 4 background workers seems like a sensible default
-    background_workers = int(os.environ.get('INVENTREE_BACKGROUND_WORKERS', 4))
-except ValueError:  # pragma: no cover
-    background_workers = 4
-
-# django-q configuration
+# django-q background worker configuration
 Q_CLUSTER = {
    'name': 'InvenTree',
-    'workers': background_workers,
-    'timeout': 90,
+    'workers': int(get_setting('INVENTREE_BACKGROUND_WORKERS', 'background.workers', 4)),
+    'timeout': int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)),
    'retry': 120,
    'queue_limit': 50,
    'bulk': 10,
@ -657,7 +559,7 @@ Q_CLUSTER = {
    'sync': False,
 }

-if _cache_host:  # pragma: no cover
+if cache_host:  # pragma: no cover
    # If using external redis cache, make the cache the broker for Django Q
    # as well
    Q_CLUSTER["django_redis"] = "worker"
@ -698,8 +600,7 @@ if type(EXTRA_URL_SCHEMES) not in [list]:  # pragma: no cover

 # Internationalization
 # https://docs.djangoproject.com/en/dev/topics/i18n/
-
-LANGUAGE_CODE = CONFIG.get('language', 'en-us')
+LANGUAGE_CODE = get_setting('INVENTREE_LANGUAGE', 'language', 'en-us')

 # If a new language translation is supported, it must be added here
 LANGUAGES = [
@ -730,7 +631,7 @@ LANGUAGES = [
 ]

 # Testing interface translations
-if get_setting('TEST_TRANSLATIONS', False):  # pragma: no cover
+if get_boolean_setting('TEST_TRANSLATIONS', default_value=False):  # pragma: no cover
    # Set default language
    LANGUAGE_CODE = 'xx'

@ -762,68 +663,29 @@ for currency in CURRENCIES:
        print(f"Currency code '{currency}' is not supported")
        sys.exit(1)

-
 # Custom currency exchange backend
 EXCHANGE_BACKEND = 'InvenTree.exchange.InvenTreeExchange'

-# Extract email settings from the config file
-email_config = CONFIG.get('email', {})
+# Email configuration options
+EMAIL_BACKEND = get_setting('INVENTREE_EMAIL_BACKEND', 'email.backend', 'django.core.mail.backends.smtp.EmailBackend')
+EMAIL_HOST = get_setting('INVENTREE_EMAIL_HOST', 'email.host', '')
+EMAIL_PORT = int(get_setting('INVENTREE_EMAIL_PORT', 'email.port', 25))
+EMAIL_HOST_USER = get_setting('INVENTREE_EMAIL_USERNAME', 'email.username', '')
+EMAIL_HOST_PASSWORD = get_setting('INVENTREE_EMAIL_PASSWORD', 'email.password', '')
+EMAIL_SUBJECT_PREFIX = get_setting('INVENTREE_EMAIL_PREFIX', 'email.prefix', '[InvenTree] ')
+EMAIL_USE_TLS = get_boolean_setting('INVENTREE_EMAIL_TLS', 'email.tls', False)
+EMAIL_USE_SSL = get_boolean_setting('INVENTREE_EMAIL_SSL', 'email.ssl', False)

-EMAIL_BACKEND = get_setting(
-    'INVENTREE_EMAIL_BACKEND',
-    email_config.get('backend', 'django.core.mail.backends.smtp.EmailBackend')
-)
-
-# Email backend settings
-EMAIL_HOST = get_setting(
-    'INVENTREE_EMAIL_HOST',
-    email_config.get('host', '')
-)
-
-EMAIL_PORT = get_setting(
-    'INVENTREE_EMAIL_PORT',
-    email_config.get('port', 25)
-)
-
-EMAIL_HOST_USER = get_setting(
-    'INVENTREE_EMAIL_USERNAME',
-    email_config.get('username', ''),
-)
-
-EMAIL_HOST_PASSWORD = get_setting(
-    'INVENTREE_EMAIL_PASSWORD',
-    email_config.get('password', ''),
-)
-
-DEFAULT_FROM_EMAIL = get_setting(
-    'INVENTREE_EMAIL_SENDER',
-    email_config.get('sender', ''),
-)
-
-EMAIL_SUBJECT_PREFIX = '[InvenTree] '
+DEFUALT_FROM_EMAIL = get_setting('INVENTREE_EMAIL_SENDER', 'email.sender', '')

 EMAIL_USE_LOCALTIME = False
-
-EMAIL_USE_TLS = get_setting(
-    'INVENTREE_EMAIL_TLS',
-    email_config.get('tls', False),
-)
-
-EMAIL_USE_SSL = get_setting(
-    'INVENTREE_EMAIL_SSL',
-    email_config.get('ssl', False),
-)
-
 EMAIL_TIMEOUT = 60

 LOCALE_PATHS = (
    BASE_DIR.joinpath('locale/'),
 )

-TIME_ZONE = get_setting(
-    'INVENTREE_TIMEZONE',
-    CONFIG.get('timezone', 'UTC')
-)
+TIME_ZONE = get_setting('INVENTREE_TIMEZONE', 'timezone', 'UTC')

 USE_I18N = True

@ -856,8 +718,8 @@ SOCIALACCOUNT_PROVIDERS = CONFIG.get('social_providers', [])
 SOCIALACCOUNT_STORE_TOKENS = True

 # settings for allauth
-ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', CONFIG.get('login_confirm_days', 3))
-ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', CONFIG.get('login_attempts', 5))
+ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', 'login_confirm_days', 3)
+ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', 'login_attempts', 5)
 ACCOUNT_LOGOUT_ON_PASSWORD_CHANGE = True
 ACCOUNT_PREVENT_ENUMERATION = True

@ -877,8 +739,8 @@ SOCIALACCOUNT_ADAPTER = 'InvenTree.forms.CustomSocialAccountAdapter'
 ACCOUNT_ADAPTER = 'InvenTree.forms.CustomAccountAdapter'

 # login settings
-REMOTE_LOGIN = get_setting('INVENTREE_REMOTE_LOGIN', CONFIG.get('remote_login', False))
-REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', CONFIG.get('remote_login_header', 'REMOTE_USER'))
+REMOTE_LOGIN = get_boolean_setting('INVENTREE_REMOTE_LOGIN', 'remote_login_enabled', False)
+REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER')

 # Markdownify configuration
 # Ref: https://django-markdownify.readthedocs.io/en/latest/settings.html
@ -909,11 +771,12 @@ MARKDOWNIFY = {
    }
 }

-# Error reporting
-SENTRY_ENABLED = get_setting('INVENTREE_SENTRY_ENABLED', CONFIG.get('sentry_enabled', False))
-SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', CONFIG.get('sentry_dsn', INVENTREE_DSN))
-
-SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', CONFIG.get('sentry_sample_rate', 0.1)))
+# sentry.io integration for error reporting
+SENTRY_ENABLED = get_boolean_setting('INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False)
+# Default Sentry DSN (can be overriden if user wants custom sentry integration)
+INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600'
+SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', 'sentry_dsn', INVENTREE_DSN)
+SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', 'sentry_sample_rate', 0.1))

 if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
    sentry_sdk.init(
@ -932,7 +795,7 @@ if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
        sentry_sdk.set_tag(f'inventree_{key}', val)

 # In-database error logging
-IGNORRED_ERRORS = [
+IGNORED_ERRORS = [
    Http404
 ]

@ -941,33 +804,29 @@ MAINTENANCE_MODE_RETRY_AFTER = 60
 MAINTENANCE_MODE_STATE_BACKEND = 'maintenance_mode.backends.DefaultStorageBackend'

 # Are plugins enabled?
-PLUGINS_ENABLED = _is_true(get_setting(
-    'INVENTREE_PLUGINS_ENABLED',
-    CONFIG.get('plugins_enabled', False),
-))
+PLUGINS_ENABLED = get_boolean_setting('INVENTREE_PLUGINS_ENABLED', 'plugins_enabled', False)

-PLUGIN_FILE = get_plugin_file()
+PLUGIN_FILE = config.get_plugin_file()

 # Plugin test settings
-PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING)  # are plugins beeing tested?
-PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False)  # load plugins from setup hooks in testing?
+PLUGIN_TESTING = CONFIG.get('PLUGIN_TESTING', TESTING)  # are plugins beeing tested?
+PLUGIN_TESTING_SETUP = CONFIG.get('PLUGIN_TESTING_SETUP', False)  # load plugins from setup hooks in testing?
 PLUGIN_TESTING_EVENTS = False                  # Flag if events are tested right now
-PLUGIN_RETRY = get_setting('PLUGIN_RETRY', 5)  # how often should plugin loading be tried?
+PLUGIN_RETRY = CONFIG.get('PLUGIN_RETRY', 5)  # how often should plugin loading be tried?
 PLUGIN_FILE_CHECKED = False                    # Was the plugin file checked?

 # User interface customization values
-CUSTOMIZE = get_setting(
-    'INVENTREE_CUSTOMIZE',
-    CONFIG.get('customize', {}),
-    {}
-)
+CUSTOMIZE = get_setting('INVENTREE_CUSTOMIZE', 'customize', {})

-CUSTOM_LOGO = get_setting(
-    'INVENTREE_CUSTOM_LOGO',
-    CUSTOMIZE.get('logo', False)
-)
+CUSTOM_LOGO = get_setting('INVENTREE_CUSTOM_LOGO', 'customize.logo', None)

 # check that the logo-file exsists in media
 if CUSTOM_LOGO and not default_storage.exists(CUSTOM_LOGO):  # pragma: no cover
    logger.warning(f"The custom logo file '{CUSTOM_LOGO}' could not be found in the default media storage")
    CUSTOM_LOGO = False
+
+if DEBUG:
+    logger.info("InvenTree running with DEBUG enabled")
+
+logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'")
+logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'")
--- a/InvenTree/InvenTree/test_middleware.py
+++ b/InvenTree/InvenTree/test_middleware.py
@ -84,7 +84,7 @@ class MiddlewareTests(InvenTreeTestCase):
            log_error('testpath')

        # Test setup without ignored errors
-        settings.IGNORRED_ERRORS = []
+        settings.IGNORED_ERRORS = []
        response = self.client.get(reverse('part-detail', kwargs={'pk': 9999}))
        self.assertEqual(response.status_code, 404)
        check(1)
--- a/InvenTree/common/migrations/0010_migrate_currency_setting.py
+++ b/InvenTree/common/migrations/0010_migrate_currency_setting.py
@ -2,12 +2,12 @@

 from django.db import migrations
 from common.models import InvenTreeSetting
-from InvenTree.settings import get_setting, CONFIG
+from InvenTree.config import get_setting

 def set_default_currency(apps, schema_editor):
    """ migrate the currency setting from config.yml to db """
    # get value from settings-file
-    base_currency = get_setting('INVENTREE_BASE_CURRENCY', CONFIG.get('base_currency', 'USD'))
+    base_currency = get_setting('INVENTREE_BASE_CURRENCY', 'base_currency', 'USD')
    # write to database
    InvenTreeSetting.set_setting('INVENTREE_DEFAULT_CURRENCY', base_currency, None, create=True)

--- a/InvenTree/config_template.yaml
+++ b/InvenTree/config_template.yaml
@ -1,7 +1,6 @@

 # Database backend selection - Configure backend database settings
-# Ref: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-DATABASES
-# Specify database parameters below as they appear in the Django docs
+# Documentation: https://inventree.readthedocs.io/en/latest/start/config/

 # Note: Database configuration options can also be specified from environmental variables,
 #       with the prefix INVENTREE_DB_
@ -44,20 +43,32 @@ database:
  # ENGINE: sqlite3
  # NAME: '/home/inventree/database.sqlite3'

+# Set debug to False to run in production mode
+# Use the environment variable INVENTREE_DEBUG
+debug: True
+
+# Configure the system logging level
+# Use environment variable INVENTREE_LOG_LEVEL
+# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL
+log_level: WARNING
+
 # Select default system language (default is 'en-us')
+# Use the environment variable INVENTREE_LANGUAGE
 language: en-us

 # System time-zone (default is UTC)
 # Reference: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-# Select an option from the "TZ database name" column
-# Use the environment variable INVENTREE_TIMEZONE
 timezone: UTC

-# Base currency code
+# Base currency code (or use env var INVENTREE_BASE_CURRENCY)
 base_currency: USD

-# List of currencies supported by default.
-# Add other currencies here to allow use in InvenTree
+# Add new user on first startup
+#admin_user: admin
+#admin_email: info@example.com
+#admin_password: inventree
+
+# List of currencies supported by default. Add other currencies here to allow use in InvenTree
 currencies:
  - AUD
  - CAD
@ -70,15 +81,6 @@ currencies:

 # Email backend configuration
 # Ref: https://docs.djangoproject.com/en/dev/topics/email/
-# Available options:
-# host: Email server host address
-# port: Email port
-# username: Account username
-# password: Account password
-# prefix: Email subject prefix
-# tls: Enable TLS support
-# ssl: Enable SSL support
-
 # Alternatively, these options can all be set using environment variables,
 # with the INVENTREE_EMAIL_ prefix:
 # e.g. INVENTREE_EMAIL_HOST / INVENTREE_EMAIL_PORT / INVENTREE_EMAIL_USERNAME
@ -94,31 +96,17 @@ email:
  tls: False
  ssl: False

-# Set debug to False to run in production mode
-# Use the environment variable INVENTREE_DEBUG
-debug: True
-
-# Set debug_toolbar to True to enable a debugging toolbar for InvenTree
-# Note: This will only be displayed if DEBUG mode is enabled,
-#       and only if InvenTree is accessed from a local IP (127.0.0.1)
-debug_toolbar: False
-
 # Set sentry_enabled to True to report errors back to the maintainers
-# Use the environment variable INVENTREE_SENTRY_ENABLED
-# sentry_enabled: True
-
-# Set sentry_dsn to your custom DSN if you want to use your own instance for error reporting
-# Use the environment variable INVENTREE_SENTRY_DSN
-# sentry_dsn: https://custom@custom.ingest.sentry.io/custom
+# Set sentry,dsn to your custom DSN if you want to use your own instance for error reporting
+sentry_enabled: False
+#sentry_sample_rate: 0.1
+#sentry_dsn: https://custom@custom.ingest.sentry.io/custom

 # Set this variable to True to enable InvenTree Plugins
 # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED
 plugins_enabled: False
-
-# Configure the system logging level
-# Use environment variable INVENTREE_LOG_LEVEL
-# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL
-log_level: WARNING
+#plugin_file: /path/to/plugins.txt
+#plugin_dir: /path/to/plugins/

 # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation)
 # A list of strings representing the host/domain names that this Django site can serve.
@ -138,14 +126,15 @@ cors:
  # - https://sub.example.com

 # MEDIA_ROOT is the local filesystem location for storing uploaded files
-# By default, it is stored under /home/inventree/data/media
-# Use environment variable INVENTREE_MEDIA_ROOT
-media_root: '/home/inventree/data/media'
+# media_root: '/home/inventree/data/media'

 # STATIC_ROOT is the local filesystem location for storing static files
-# By default, it is stored under /home/inventree/data/static
-# Use environment variable INVENTREE_STATIC_ROOT
-static_root: '/home/inventree/data/static'
+# static_root: '/home/inventree/data/static'
+
+# Background worker options
+background:
+  workers: 4
+  timeout: 90

 # Optional URL schemes to allow in URL fields
 # By default, only the following schemes are allowed: ['http', 'https', 'ftp', 'ftps']
@ -156,25 +145,14 @@ static_root: '/home/inventree/data/static'
 #  - ssh

 # Login configuration
-# How long do confirmation mail last?
-# Use environment variable INVENTREE_LOGIN_CONFIRM_DAYS
-#login_confirm_days: 3
-# How many wrong login attempts are permitted?
-# Use environment variable INVENTREE_LOGIN_ATTEMPTS
-#login_attempts: 5
+login_confirm_days: 3
+login_attempts: 5

 # Remote / proxy login
 # These settings can introduce security problems if configured incorrectly. Please read
 # https://docs.djangoproject.com/en/4.0/howto/auth-remote-user/ for more details
-# Use environment variable INVENTREE_REMOTE_LOGIN
-# remote_login: True
-# Use environment variable INVENTREE_REMOTE_LOGIN_HEADER
-# remote_login_header: REMOTE_USER
-
-# Add new user on first startup
-#admin_user: admin
-#admin_email: info@example.com
-#admin_password: inventree
+remote_login_enabled: False
+remote_login_header: REMOTE_USER

 # Permit custom authentication backends
 #authentication_backends:
--- a/InvenTree/plugin/registry.py
+++ b/InvenTree/plugin/registry.py
@ -22,6 +22,8 @@ from django.utils.text import slugify
 from maintenance_mode.core import (get_maintenance_mode, maintenance_mode_on,
                                   set_maintenance_mode)

+from InvenTree.config import get_setting
+
 from .helpers import (IntegrationPluginError, get_plugins, handle_error,
                      log_error)
 from .plugin import InvenTreePlugin
@ -199,7 +201,7 @@ class PluginsRegistry:
        if settings.TESTING:
            custom_dirs = os.getenv('INVENTREE_PLUGIN_TEST_DIR', None)
        else:
-            custom_dirs = os.getenv('INVENTREE_PLUGIN_DIR', None)
+            custom_dirs = get_setting('INVENTREE_PLUGIN_DIR', 'plugin_dir')

            # Load from user specified directories (unless in testing mode)
            dirs.append('plugins')
--- a/docker/requirements.txt
+++ b/docker/requirements.txt
@ -1,9 +1,10 @@
 # Base python requirements for docker containers

 # Basic package requirements
+invoke>=1.4.0                   # Invoke build tool
+pyyaml>=6.0
 setuptools==60.0.5
 wheel>=0.37.0
-invoke>=1.4.0                   # Invoke build tool

 # Database links
 psycopg2>=2.9.1
--- a/tasks.py
+++ b/tasks.py
@ -4,6 +4,7 @@ import json
 import os
 import pathlib
 import re
+import shutil
 import sys
 from pathlib import Path

@ -522,6 +523,8 @@ def test(c, database=None):
 def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset"):
    """Setup a testing enviroment."""

+    from InvenTree.InvenTree.config import get_media_dir
+
    if not ignore_update:
        update(c)

@ -540,8 +543,16 @@ def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset")
        migrate(c)

    # Load data
-    print("Loading data ...")
+    print("Loading database records ...")
    import_records(c, filename=f'{path}/inventree_data.json', clear=True)
+
+    # Copy media files
+    print("Copying media files ...")
+    src = Path(path).joinpath('media').resolve()
+    dst = get_media_dir()
+
+    shutil.copytree(src, dst, dirs_exist_ok=True)
+
    print("Done setting up test enviroment...")
    print("========================================")