diff --git a/.gitignore b/.gitignore index 6bbfdaf23d..693ed10362 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,6 @@ docs/_build inventree_media inventree_static static_i18n -inventree-data # Local config file config.yaml @@ -79,6 +78,8 @@ js_tmp/ # Development files dev/ +data/ +env/ # Locale stats file locale_stats.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd60026f42..c736418892 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,7 @@ pip install invoke && invoke setup-dev --tests ```bash git clone https://github.com/inventree/InvenTree.git && cd InvenTree +docker compose run inventree-dev-server invoke install docker compose run inventree-dev-server invoke setup-test docker compose up -d ``` diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index 3ec084a2d1..7190197104 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -172,21 +172,13 @@ class InvenTreeConfig(AppConfig): return # get values - add_user = get_setting( - 'INVENTREE_ADMIN_USER', - settings.CONFIG.get('admin_user', False) - ) - add_email = get_setting( - 'INVENTREE_ADMIN_EMAIL', - settings.CONFIG.get('admin_email', False) - ) - add_password = get_setting( - 'INVENTREE_ADMIN_PASSWORD', - settings.CONFIG.get('admin_password', False) - ) + add_user = get_setting('INVENTREE_ADMIN_USER', 'admin_user') + add_email = get_setting('INVENTREE_ADMIN_EMAIL', 'admin_email') + add_password = get_setting('INVENTREE_ADMIN_PASSWORD', 'admin_password') # check if all values are present set_variables = 0 + for tested_var in [add_user, add_email, add_password]: if tested_var: set_variables += 1 diff --git a/InvenTree/InvenTree/config.py b/InvenTree/InvenTree/config.py index e2fbe680cd..799788a0da 100644 --- a/InvenTree/InvenTree/config.py +++ b/InvenTree/InvenTree/config.py @@ -2,18 +2,27 @@ import logging import os +import random import shutil +import string from pathlib import Path +import yaml + logger = logging.getLogger('inventree') +def is_true(x): + """Shortcut function to determine if a value "looks" like a boolean""" + return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true', 'on'] + + def get_base_dir() -> Path: """Returns the base (top-level) InvenTree directory.""" return Path(__file__).parent.parent.resolve() -def get_config_file() -> Path: +def get_config_file(create=True) -> Path: """Returns the path of the InvenTree configuration file. Note: It will be created it if does not already exist! @@ -28,7 +37,7 @@ def get_config_file() -> Path: # Config file is *not* specified - use the default cfg_filename = base_dir.joinpath('config.yaml').resolve() - if not cfg_filename.exists(): + if not cfg_filename.exists() and create: print("InvenTree configuration file 'config.yaml' not found - creating default file") cfg_template = base_dir.joinpath("config_template.yaml") @@ -38,45 +47,159 @@ def get_config_file() -> Path: return cfg_filename -def get_plugin_file(): - """Returns the path of the InvenTree plugins specification file. +def load_config_data() -> map: + """Load configuration data from the config file.""" - Note: It will be created if it does not already exist! - """ - # Check if the plugin.txt file (specifying required plugins) is specified - PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') + cfg_file = get_config_file() - if not PLUGIN_FILE: - # If not specified, look in the same directory as the configuration file - config_dir = get_config_file().parent - PLUGIN_FILE = config_dir.joinpath('plugins.txt') - else: - # Make sure we are using a modern Path object - PLUGIN_FILE = Path(PLUGIN_FILE) + with open(cfg_file, 'r') as cfg: + data = yaml.safe_load(cfg) - if not PLUGIN_FILE.exists(): - logger.warning("Plugin configuration file does not exist") - logger.info(f"Creating plugin file at '{PLUGIN_FILE}'") - - # If opening the file fails (no write permission, for example), then this will throw an error - PLUGIN_FILE.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n") - - return PLUGIN_FILE + return data -def get_setting(environment_var, backup_val, default_value=None): +def get_setting(env_var=None, config_key=None, default_value=None): """Helper function for retrieving a configuration setting value. - First preference is to look for the environment variable - Second preference is to look for the value of the settings file - Third preference is the default value + + Arguments: + env_var: Name of the environment variable e.g. 'INVENTREE_STATIC_ROOT' + config_key: Key to lookup in the configuration file + default_value: Value to return if first two options are not provided + """ - val = os.getenv(environment_var) - if val is not None: - return val + # First, try to load from the environment variables + if env_var is not None: + val = os.getenv(env_var, None) - if backup_val is not None: - return backup_val + if val is not None: + return val + # Next, try to load from configuration file + if config_key is not None: + cfg_data = load_config_data() + + result = None + + # Hack to allow 'path traversal' in configuration file + for key in config_key.strip().split('.'): + + if type(cfg_data) is not dict or key not in cfg_data: + result = None + break + + result = cfg_data[key] + cfg_data = cfg_data[key] + + if result is not None: + return result + + # Finally, return the default value return default_value + + +def get_boolean_setting(env_var=None, config_key=None, default_value=False): + """Helper function for retreiving a boolean configuration setting""" + + return is_true(get_setting(env_var, config_key, default_value)) + + +def get_media_dir(create=True): + """Return the absolute path for the 'media' directory (where uploaded files are stored)""" + + md = get_setting('INVENTREE_MEDIA_ROOT', 'media_root') + + if not md: + raise FileNotFoundError('INVENTREE_MEDIA_ROOT not specified') + + md = Path(md).resolve() + + if create: + md.mkdir(parents=True, exist_ok=True) + + return md + + +def get_static_dir(create=True): + """Return the absolute path for the 'static' directory (where static files are stored)""" + + sd = get_setting('INVENTREE_STATIC_ROOT', 'static_root') + + if not sd: + raise FileNotFoundError('INVENTREE_STATIC_ROOT not specified') + + sd = Path(sd).resolve() + + if create: + sd.mkdir(parents=True, exist_ok=True) + + return sd + + +def get_plugin_file(): + """Returns the path of the InvenTree plugins specification file. + + Note: It will be created if it does not already exist! + """ + + # Check if the plugin.txt file (specifying required plugins) is specified + plugin_file = get_setting('INVENTREE_PLUGIN_FILE', 'plugin_file') + + if not plugin_file: + # If not specified, look in the same directory as the configuration file + config_dir = get_config_file().parent + plugin_file = config_dir.joinpath('plugins.txt') + else: + # Make sure we are using a modern Path object + plugin_file = Path(plugin_file) + + if not plugin_file.exists(): + logger.warning("Plugin configuration file does not exist - creating default file") + logger.info(f"Creating plugin file at '{plugin_file}'") + + # If opening the file fails (no write permission, for example), then this will throw an error + plugin_file.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n") + + return plugin_file + + +def get_secret_key(): + """Return the secret key value which will be used by django. + + Following options are tested, in descending order of preference: + + A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data + B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file + C) Look for default key file "secret_key.txt" + D) Create "secret_key.txt" if it does not exist + """ + + # Look for environment variable + if secret_key := get_setting('INVENTREE_SECRET_KEY', 'secret_key'): + logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY") # pragma: no cover + return secret_key + + # Look for secret key file + if secret_key_file := get_setting('INVENTREE_SECRET_KEY_FILE', 'secret_key_file'): + secret_key_file = Path(secret_key_file).resolve() + else: + # Default location for secret key file + secret_key_file = get_base_dir().joinpath("secret_key.txt").resolve() + + if not secret_key_file.exists(): + logger.info(f"Generating random key file at '{secret_key_file}'") + + # Create a random key file + options = string.digits + string.ascii_letters + string.punctuation + key = ''.join([random.choice(options) for i in range(100)]) + secret_key_file.write_text(key) + + logger.info(f"Loading SECRET_KEY from '{secret_key_file}'") + + key_data = secret_key_file.read_text().strip() + + return key_data diff --git a/InvenTree/InvenTree/exceptions.py b/InvenTree/InvenTree/exceptions.py index 780359d88b..2dfda365c5 100644 --- a/InvenTree/InvenTree/exceptions.py +++ b/InvenTree/InvenTree/exceptions.py @@ -29,7 +29,7 @@ def log_error(path): kind, info, data = sys.exc_info() # Check if the eror is on the ignore list - if kind in settings.IGNORRED_ERRORS: + if kind in settings.IGNORED_ERRORS: return Error.objects.create( diff --git a/InvenTree/InvenTree/middleware.py b/InvenTree/InvenTree/middleware.py index 1e282265c5..951d415c16 100644 --- a/InvenTree/InvenTree/middleware.py +++ b/InvenTree/InvenTree/middleware.py @@ -157,7 +157,7 @@ class InvenTreeExceptionProcessor(ExceptionProcessor): kind, info, data = sys.exc_info() # Check if the eror is on the ignore list - if kind in settings.IGNORRED_ERRORS: + if kind in settings.IGNORED_ERRORS: return return super().process_exception(request, exception) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index f9cf41deba..4f763b0089 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -11,9 +11,7 @@ database setup in this file. import logging import os -import random import socket -import string import sys from pathlib import Path @@ -24,22 +22,14 @@ from django.utils.translation import gettext_lazy as _ import moneyed import sentry_sdk -import yaml from sentry_sdk.integrations.django import DjangoIntegration -from .config import get_base_dir, get_config_file, get_plugin_file, get_setting - - -def _is_true(x): - # Shortcut function to determine if a value "looks" like a boolean - return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true'] - - -# Default Sentry DSN (can be overriden if user wants custom sentry integration) -INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600' +from . import config +from .config import get_boolean_setting, get_setting # Determine if we are running in "test" mode e.g. "manage.py test" TESTING = 'test' in sys.argv + # Are enviroment variables manipulated by tests? Needs to be set by testing code TESTING_ENV = False @@ -47,33 +37,17 @@ TESTING_ENV = False DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' # Build paths inside the project like this: BASE_DIR.joinpath(...) -BASE_DIR = get_base_dir() +BASE_DIR = config.get_base_dir() -cfg_filename = get_config_file() - -with open(cfg_filename, 'r') as cfg: - CONFIG = yaml.safe_load(cfg) - -# We will place any config files in the same directory as the config file -config_dir = cfg_filename.parent +# Load configuration data +CONFIG = config.load_config_data() # Default action is to run the system in Debug mode # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = _is_true(get_setting( - 'INVENTREE_DEBUG', - CONFIG.get('debug', True) -)) - -DOCKER = _is_true(get_setting( - 'INVENTREE_DOCKER', - False -)) +DEBUG = get_boolean_setting('INVENTREE_DEBUG', 'debug', True) # Configure logging settings -log_level = get_setting( - 'INVENTREE_LOG_LEVEL', - CONFIG.get('log_level', 'WARNING') -) +log_level = get_setting('INVENTREE_LOG_LEVEL', 'log_level', 'WARNING') logging.basicConfig( level=log_level, @@ -105,78 +79,20 @@ LOGGING = { # Get a logger instance for this setup file logger = logging.getLogger("inventree") -""" -Specify a secret key to be used by django. - -Following options are tested, in descending order of preference: - -A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data -B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file -C) Look for default key file "secret_key.txt" -d) Create "secret_key.txt" if it does not exist -""" - -if secret_key := os.getenv("INVENTREE_SECRET_KEY"): - # Secret key passed in directly - SECRET_KEY = secret_key.strip() # pragma: no cover - logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY") # pragma: no cover -else: - # Secret key passed in by file location - key_file = os.getenv("INVENTREE_SECRET_KEY_FILE") - - if key_file: - key_file = Path(key_file).resolve() # pragma: no cover - else: - # default secret key location - key_file = BASE_DIR.joinpath("secret_key.txt").resolve() - - if not key_file.exists(): # pragma: no cover - logger.info(f"Generating random key file at '{key_file}'") - # Create a random key file - options = string.digits + string.ascii_letters + string.punctuation - key = ''.join([random.choice(options) for i in range(100)]) - key_file.write_text(key) - - logger.info(f"Loading SECRET_KEY from '{key_file}'") - - try: - SECRET_KEY = open(key_file, "r").read().strip() - except Exception: # pragma: no cover - logger.exception(f"Couldn't load keyfile {key_file}") - sys.exit(-1) +# Load SECRET_KEY +SECRET_KEY = config.get_secret_key() # The filesystem location for served static files -STATIC_ROOT = Path( - get_setting( - 'INVENTREE_STATIC_ROOT', - CONFIG.get('static_root', None) - ) -).resolve() +STATIC_ROOT = config.get_static_dir() -if STATIC_ROOT is None: # pragma: no cover - print("ERROR: INVENTREE_STATIC_ROOT directory not defined") - sys.exit(1) -else: - # Ensure the root really is availalble - STATIC_ROOT.mkdir(parents=True, exist_ok=True) - -# The filesystem location for served static files -MEDIA_ROOT = Path( - get_setting( - 'INVENTREE_MEDIA_ROOT', - CONFIG.get('media_root', None) - ) -).resolve() - -if MEDIA_ROOT is None: # pragma: no cover - print("ERROR: INVENTREE_MEDIA_ROOT directory is not defined") - sys.exit(1) -else: - # Ensure the root really is availalble - MEDIA_ROOT.mkdir(parents=True, exist_ok=True) +# The filesystem location for uploaded meadia files +MEDIA_ROOT = config.get_media_dir() # List of allowed hosts (default = allow all) -ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*']) +ALLOWED_HOSTS = get_setting( + config_key='allowed_hosts', + default_value=['*'] +) # Cross Origin Resource Sharing (CORS) options @@ -184,13 +100,15 @@ ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*']) CORS_URLS_REGEX = r'^/api/.*$' # Extract CORS options from configuration file -cors_opt = CONFIG.get('cors', None) +CORS_ORIGIN_ALLOW_ALL = get_boolean_setting( + config_key='cors.allow_all', + default_value=False, +) -if cors_opt: - CORS_ORIGIN_ALLOW_ALL = cors_opt.get('allow_all', False) - - if not CORS_ORIGIN_ALLOW_ALL: - CORS_ORIGIN_WHITELIST = cors_opt.get('whitelist', []) # pragma: no cover +CORS_ORIGIN_WHITELIST = get_setting( + config_key='cors.whitelist', + default_value=[] +) # Web URL endpoint for served static files STATIC_URL = '/static/' @@ -214,12 +132,6 @@ STATIC_COLOR_THEMES_DIR = STATIC_ROOT.joinpath('css', 'color-themes') # Web URL endpoint for served media files MEDIA_URL = '/media/' -if DEBUG: - logger.info("InvenTree running with DEBUG enabled") - -logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'") -logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'") - # Application definition INSTALLED_APPS = [ @@ -320,6 +232,9 @@ INTERNAL_IPS = [ '127.0.0.1', ] +# Internal flag to determine if we are running in docker mode +DOCKER = get_boolean_setting('INVENTREE_DOCKER', default_value=False) + if DOCKER: # pragma: no cover # Internal IP addresses are different when running under docker hostname, ___, ips = socket.gethostbyname_ex(socket.gethostname()) @@ -334,7 +249,8 @@ if DEBUG: # Base URL for admin pages (default="admin") INVENTREE_ADMIN_URL = get_setting( 'INVENTREE_ADMIN_URL', - CONFIG.get('admin_url', 'admin'), + config_key='admin_url', + default_value='admin' ) ROOT_URLCONF = 'InvenTree.urls' @@ -498,7 +414,7 @@ if "postgres" in db_engine: # pragma: no cover # long to connect to the database server # # seconds, 2 is minium allowed by libpq db_options["connect_timeout"] = int( - os.getenv("INVENTREE_DB_TIMEOUT", 2) + get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 2) ) # Setup TCP keepalive @@ -509,23 +425,27 @@ if "postgres" in db_engine: # pragma: no cover # # 0 - TCP Keepalives disabled; 1 - enabled if "keepalives" not in db_options: db_options["keepalives"] = int( - os.getenv("INVENTREE_DB_TCP_KEEPALIVES", "1") + get_setting('INVENTREE_DB_TCP_KEEPALIVES', 'database.tcp_keepalives', 1) ) - # # Seconds after connection is idle to send keep alive + + # Seconds after connection is idle to send keep alive if "keepalives_idle" not in db_options: db_options["keepalives_idle"] = int( - os.getenv("INVENTREE_DB_TCP_KEEPALIVES_IDLE", "1") + get_setting('INVENTREE_DB_TCP_KEEPALIVES_IDLE', 'database.tcp_keepalives_idle', 1) ) - # # Seconds after missing ACK to send another keep alive + + # Seconds after missing ACK to send another keep alive if "keepalives_interval" not in db_options: db_options["keepalives_interval"] = int( - os.getenv("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "1") + get_setting("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "database.tcp_keepalives_internal", "1") ) - # # Number of missing ACKs before we close the connection + + # Number of missing ACKs before we close the connection if "keepalives_count" not in db_options: db_options["keepalives_count"] = int( - os.getenv("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "5") + get_setting("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "database.tcp_keepalives_count", "5") ) + # # Milliseconds for how long pending data should remain unacked # by the remote server # TODO: Supported starting in PSQL 11 @@ -538,17 +458,11 @@ if "postgres" in db_engine: # pragma: no cover # https://www.postgresql.org/docs/devel/transaction-iso.html # https://docs.djangoproject.com/en/3.2/ref/databases/#isolation-level if "isolation_level" not in db_options: - serializable = _is_true( - os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false") - ) - db_options["isolation_level"] = ( - ISOLATION_LEVEL_SERIALIZABLE - if serializable - else ISOLATION_LEVEL_READ_COMMITTED - ) + serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False) + db_options["isolation_level"] = ISOLATION_LEVEL_SERIALIZABLE if serializable else ISOLATION_LEVEL_READ_COMMITTED # Specific options for MySql / MariaDB backend -if "mysql" in db_engine: # pragma: no cover +elif "mysql" in db_engine: # pragma: no cover # TODO TCP time outs and keepalives # MariaDB's default isolation level is Repeatable Read which is @@ -558,15 +472,11 @@ if "mysql" in db_engine: # pragma: no cover # https://mariadb.com/kb/en/mariadb-transactions-and-isolation-levels-for-sql-server-users/#changing-the-isolation-level # https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level if "isolation_level" not in db_options: - serializable = _is_true( - os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false") - ) - db_options["isolation_level"] = ( - "serializable" if serializable else "read committed" - ) + serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False) + db_options["isolation_level"] = "serializable" if serializable else "read committed" # Specific options for sqlite backend -if "sqlite" in db_engine: +elif "sqlite" in db_engine: # TODO: Verify timeouts are not an issue because no network is involved for SQLite # SQLite's default isolation level is Serializable due to SQLite's @@ -591,13 +501,11 @@ DATABASES = { 'default': db_config } -_cache_config = CONFIG.get("cache", {}) -_cache_host = _cache_config.get("host", os.getenv("INVENTREE_CACHE_HOST")) -_cache_port = _cache_config.get( - "port", os.getenv("INVENTREE_CACHE_PORT", "6379") -) +# Cache configuration +cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None) +cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379') -if _cache_host: # pragma: no cover +if cache_host: # pragma: no cover # We are going to rely upon a possibly non-localhost for our cache, # so don't wait too long for the cache as nothing in the cache should be # irreplacable. @@ -606,7 +514,7 @@ if _cache_host: # pragma: no cover "SOCKET_CONNECT_TIMEOUT": int(os.getenv("CACHE_CONNECT_TIMEOUT", "2")), "SOCKET_TIMEOUT": int(os.getenv("CACHE_SOCKET_TIMEOUT", "2")), "CONNECTION_POOL_KWARGS": { - "socket_keepalive": _is_true( + "socket_keepalive": config.is_true( os.getenv("CACHE_TCP_KEEPALIVE", "1") ), "socket_keepalive_options": { @@ -628,7 +536,7 @@ if _cache_host: # pragma: no cover CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://{_cache_host}:{_cache_port}/0", + "LOCATION": f"redis://{cache_host}:{cache_port}/0", "OPTIONS": _cache_options, }, } @@ -639,17 +547,11 @@ else: }, } -try: - # 4 background workers seems like a sensible default - background_workers = int(os.environ.get('INVENTREE_BACKGROUND_WORKERS', 4)) -except ValueError: # pragma: no cover - background_workers = 4 - -# django-q configuration +# django-q background worker configuration Q_CLUSTER = { 'name': 'InvenTree', - 'workers': background_workers, - 'timeout': 90, + 'workers': int(get_setting('INVENTREE_BACKGROUND_WORKERS', 'background.workers', 4)), + 'timeout': int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)), 'retry': 120, 'queue_limit': 50, 'bulk': 10, @@ -657,7 +559,7 @@ Q_CLUSTER = { 'sync': False, } -if _cache_host: # pragma: no cover +if cache_host: # pragma: no cover # If using external redis cache, make the cache the broker for Django Q # as well Q_CLUSTER["django_redis"] = "worker" @@ -698,8 +600,7 @@ if type(EXTRA_URL_SCHEMES) not in [list]: # pragma: no cover # Internationalization # https://docs.djangoproject.com/en/dev/topics/i18n/ - -LANGUAGE_CODE = CONFIG.get('language', 'en-us') +LANGUAGE_CODE = get_setting('INVENTREE_LANGUAGE', 'language', 'en-us') # If a new language translation is supported, it must be added here LANGUAGES = [ @@ -730,7 +631,7 @@ LANGUAGES = [ ] # Testing interface translations -if get_setting('TEST_TRANSLATIONS', False): # pragma: no cover +if get_boolean_setting('TEST_TRANSLATIONS', default_value=False): # pragma: no cover # Set default language LANGUAGE_CODE = 'xx' @@ -762,68 +663,29 @@ for currency in CURRENCIES: print(f"Currency code '{currency}' is not supported") sys.exit(1) - # Custom currency exchange backend EXCHANGE_BACKEND = 'InvenTree.exchange.InvenTreeExchange' -# Extract email settings from the config file -email_config = CONFIG.get('email', {}) +# Email configuration options +EMAIL_BACKEND = get_setting('INVENTREE_EMAIL_BACKEND', 'email.backend', 'django.core.mail.backends.smtp.EmailBackend') +EMAIL_HOST = get_setting('INVENTREE_EMAIL_HOST', 'email.host', '') +EMAIL_PORT = int(get_setting('INVENTREE_EMAIL_PORT', 'email.port', 25)) +EMAIL_HOST_USER = get_setting('INVENTREE_EMAIL_USERNAME', 'email.username', '') +EMAIL_HOST_PASSWORD = get_setting('INVENTREE_EMAIL_PASSWORD', 'email.password', '') +EMAIL_SUBJECT_PREFIX = get_setting('INVENTREE_EMAIL_PREFIX', 'email.prefix', '[InvenTree] ') +EMAIL_USE_TLS = get_boolean_setting('INVENTREE_EMAIL_TLS', 'email.tls', False) +EMAIL_USE_SSL = get_boolean_setting('INVENTREE_EMAIL_SSL', 'email.ssl', False) -EMAIL_BACKEND = get_setting( - 'INVENTREE_EMAIL_BACKEND', - email_config.get('backend', 'django.core.mail.backends.smtp.EmailBackend') -) - -# Email backend settings -EMAIL_HOST = get_setting( - 'INVENTREE_EMAIL_HOST', - email_config.get('host', '') -) - -EMAIL_PORT = get_setting( - 'INVENTREE_EMAIL_PORT', - email_config.get('port', 25) -) - -EMAIL_HOST_USER = get_setting( - 'INVENTREE_EMAIL_USERNAME', - email_config.get('username', ''), -) - -EMAIL_HOST_PASSWORD = get_setting( - 'INVENTREE_EMAIL_PASSWORD', - email_config.get('password', ''), -) - -DEFAULT_FROM_EMAIL = get_setting( - 'INVENTREE_EMAIL_SENDER', - email_config.get('sender', ''), -) - -EMAIL_SUBJECT_PREFIX = '[InvenTree] ' +DEFUALT_FROM_EMAIL = get_setting('INVENTREE_EMAIL_SENDER', 'email.sender', '') EMAIL_USE_LOCALTIME = False - -EMAIL_USE_TLS = get_setting( - 'INVENTREE_EMAIL_TLS', - email_config.get('tls', False), -) - -EMAIL_USE_SSL = get_setting( - 'INVENTREE_EMAIL_SSL', - email_config.get('ssl', False), -) - EMAIL_TIMEOUT = 60 LOCALE_PATHS = ( BASE_DIR.joinpath('locale/'), ) -TIME_ZONE = get_setting( - 'INVENTREE_TIMEZONE', - CONFIG.get('timezone', 'UTC') -) +TIME_ZONE = get_setting('INVENTREE_TIMEZONE', 'timezone', 'UTC') USE_I18N = True @@ -856,8 +718,8 @@ SOCIALACCOUNT_PROVIDERS = CONFIG.get('social_providers', []) SOCIALACCOUNT_STORE_TOKENS = True # settings for allauth -ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', CONFIG.get('login_confirm_days', 3)) -ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', CONFIG.get('login_attempts', 5)) +ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', 'login_confirm_days', 3) +ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', 'login_attempts', 5) ACCOUNT_LOGOUT_ON_PASSWORD_CHANGE = True ACCOUNT_PREVENT_ENUMERATION = True @@ -877,8 +739,8 @@ SOCIALACCOUNT_ADAPTER = 'InvenTree.forms.CustomSocialAccountAdapter' ACCOUNT_ADAPTER = 'InvenTree.forms.CustomAccountAdapter' # login settings -REMOTE_LOGIN = get_setting('INVENTREE_REMOTE_LOGIN', CONFIG.get('remote_login', False)) -REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', CONFIG.get('remote_login_header', 'REMOTE_USER')) +REMOTE_LOGIN = get_boolean_setting('INVENTREE_REMOTE_LOGIN', 'remote_login_enabled', False) +REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER') # Markdownify configuration # Ref: https://django-markdownify.readthedocs.io/en/latest/settings.html @@ -909,11 +771,12 @@ MARKDOWNIFY = { } } -# Error reporting -SENTRY_ENABLED = get_setting('INVENTREE_SENTRY_ENABLED', CONFIG.get('sentry_enabled', False)) -SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', CONFIG.get('sentry_dsn', INVENTREE_DSN)) - -SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', CONFIG.get('sentry_sample_rate', 0.1))) +# sentry.io integration for error reporting +SENTRY_ENABLED = get_boolean_setting('INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False) +# Default Sentry DSN (can be overriden if user wants custom sentry integration) +INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600' +SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', 'sentry_dsn', INVENTREE_DSN) +SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', 'sentry_sample_rate', 0.1)) if SENTRY_ENABLED and SENTRY_DSN: # pragma: no cover sentry_sdk.init( @@ -932,7 +795,7 @@ if SENTRY_ENABLED and SENTRY_DSN: # pragma: no cover sentry_sdk.set_tag(f'inventree_{key}', val) # In-database error logging -IGNORRED_ERRORS = [ +IGNORED_ERRORS = [ Http404 ] @@ -941,33 +804,29 @@ MAINTENANCE_MODE_RETRY_AFTER = 60 MAINTENANCE_MODE_STATE_BACKEND = 'maintenance_mode.backends.DefaultStorageBackend' # Are plugins enabled? -PLUGINS_ENABLED = _is_true(get_setting( - 'INVENTREE_PLUGINS_ENABLED', - CONFIG.get('plugins_enabled', False), -)) +PLUGINS_ENABLED = get_boolean_setting('INVENTREE_PLUGINS_ENABLED', 'plugins_enabled', False) -PLUGIN_FILE = get_plugin_file() +PLUGIN_FILE = config.get_plugin_file() # Plugin test settings -PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING) # are plugins beeing tested? -PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False) # load plugins from setup hooks in testing? +PLUGIN_TESTING = CONFIG.get('PLUGIN_TESTING', TESTING) # are plugins beeing tested? +PLUGIN_TESTING_SETUP = CONFIG.get('PLUGIN_TESTING_SETUP', False) # load plugins from setup hooks in testing? PLUGIN_TESTING_EVENTS = False # Flag if events are tested right now -PLUGIN_RETRY = get_setting('PLUGIN_RETRY', 5) # how often should plugin loading be tried? +PLUGIN_RETRY = CONFIG.get('PLUGIN_RETRY', 5) # how often should plugin loading be tried? PLUGIN_FILE_CHECKED = False # Was the plugin file checked? # User interface customization values -CUSTOMIZE = get_setting( - 'INVENTREE_CUSTOMIZE', - CONFIG.get('customize', {}), - {} -) +CUSTOMIZE = get_setting('INVENTREE_CUSTOMIZE', 'customize', {}) -CUSTOM_LOGO = get_setting( - 'INVENTREE_CUSTOM_LOGO', - CUSTOMIZE.get('logo', False) -) +CUSTOM_LOGO = get_setting('INVENTREE_CUSTOM_LOGO', 'customize.logo', None) # check that the logo-file exsists in media if CUSTOM_LOGO and not default_storage.exists(CUSTOM_LOGO): # pragma: no cover logger.warning(f"The custom logo file '{CUSTOM_LOGO}' could not be found in the default media storage") CUSTOM_LOGO = False + +if DEBUG: + logger.info("InvenTree running with DEBUG enabled") + +logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'") +logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'") diff --git a/InvenTree/InvenTree/test_middleware.py b/InvenTree/InvenTree/test_middleware.py index 5f9eb3b9c4..fcfc4fe933 100644 --- a/InvenTree/InvenTree/test_middleware.py +++ b/InvenTree/InvenTree/test_middleware.py @@ -84,7 +84,7 @@ class MiddlewareTests(InvenTreeTestCase): log_error('testpath') # Test setup without ignored errors - settings.IGNORRED_ERRORS = [] + settings.IGNORED_ERRORS = [] response = self.client.get(reverse('part-detail', kwargs={'pk': 9999})) self.assertEqual(response.status_code, 404) check(1) diff --git a/InvenTree/common/migrations/0010_migrate_currency_setting.py b/InvenTree/common/migrations/0010_migrate_currency_setting.py index 23076ff200..0604e93008 100644 --- a/InvenTree/common/migrations/0010_migrate_currency_setting.py +++ b/InvenTree/common/migrations/0010_migrate_currency_setting.py @@ -2,12 +2,12 @@ from django.db import migrations from common.models import InvenTreeSetting -from InvenTree.settings import get_setting, CONFIG +from InvenTree.config import get_setting def set_default_currency(apps, schema_editor): """ migrate the currency setting from config.yml to db """ # get value from settings-file - base_currency = get_setting('INVENTREE_BASE_CURRENCY', CONFIG.get('base_currency', 'USD')) + base_currency = get_setting('INVENTREE_BASE_CURRENCY', 'base_currency', 'USD') # write to database InvenTreeSetting.set_setting('INVENTREE_DEFAULT_CURRENCY', base_currency, None, create=True) diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index a470ac62e2..aa5d715d04 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -1,7 +1,6 @@ # Database backend selection - Configure backend database settings -# Ref: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-DATABASES -# Specify database parameters below as they appear in the Django docs +# Documentation: https://inventree.readthedocs.io/en/latest/start/config/ # Note: Database configuration options can also be specified from environmental variables, # with the prefix INVENTREE_DB_ @@ -44,20 +43,32 @@ database: # ENGINE: sqlite3 # NAME: '/home/inventree/database.sqlite3' +# Set debug to False to run in production mode +# Use the environment variable INVENTREE_DEBUG +debug: True + +# Configure the system logging level +# Use environment variable INVENTREE_LOG_LEVEL +# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL +log_level: WARNING + # Select default system language (default is 'en-us') +# Use the environment variable INVENTREE_LANGUAGE language: en-us # System time-zone (default is UTC) # Reference: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -# Select an option from the "TZ database name" column -# Use the environment variable INVENTREE_TIMEZONE timezone: UTC -# Base currency code +# Base currency code (or use env var INVENTREE_BASE_CURRENCY) base_currency: USD -# List of currencies supported by default. -# Add other currencies here to allow use in InvenTree +# Add new user on first startup +#admin_user: admin +#admin_email: info@example.com +#admin_password: inventree + +# List of currencies supported by default. Add other currencies here to allow use in InvenTree currencies: - AUD - CAD @@ -70,15 +81,6 @@ currencies: # Email backend configuration # Ref: https://docs.djangoproject.com/en/dev/topics/email/ -# Available options: -# host: Email server host address -# port: Email port -# username: Account username -# password: Account password -# prefix: Email subject prefix -# tls: Enable TLS support -# ssl: Enable SSL support - # Alternatively, these options can all be set using environment variables, # with the INVENTREE_EMAIL_ prefix: # e.g. INVENTREE_EMAIL_HOST / INVENTREE_EMAIL_PORT / INVENTREE_EMAIL_USERNAME @@ -94,31 +96,17 @@ email: tls: False ssl: False -# Set debug to False to run in production mode -# Use the environment variable INVENTREE_DEBUG -debug: True - -# Set debug_toolbar to True to enable a debugging toolbar for InvenTree -# Note: This will only be displayed if DEBUG mode is enabled, -# and only if InvenTree is accessed from a local IP (127.0.0.1) -debug_toolbar: False - # Set sentry_enabled to True to report errors back to the maintainers -# Use the environment variable INVENTREE_SENTRY_ENABLED -# sentry_enabled: True - -# Set sentry_dsn to your custom DSN if you want to use your own instance for error reporting -# Use the environment variable INVENTREE_SENTRY_DSN -# sentry_dsn: https://custom@custom.ingest.sentry.io/custom +# Set sentry,dsn to your custom DSN if you want to use your own instance for error reporting +sentry_enabled: False +#sentry_sample_rate: 0.1 +#sentry_dsn: https://custom@custom.ingest.sentry.io/custom # Set this variable to True to enable InvenTree Plugins # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED plugins_enabled: False - -# Configure the system logging level -# Use environment variable INVENTREE_LOG_LEVEL -# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL -log_level: WARNING +#plugin_file: /path/to/plugins.txt +#plugin_dir: /path/to/plugins/ # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation) # A list of strings representing the host/domain names that this Django site can serve. @@ -138,14 +126,15 @@ cors: # - https://sub.example.com # MEDIA_ROOT is the local filesystem location for storing uploaded files -# By default, it is stored under /home/inventree/data/media -# Use environment variable INVENTREE_MEDIA_ROOT -media_root: '/home/inventree/data/media' +# media_root: '/home/inventree/data/media' # STATIC_ROOT is the local filesystem location for storing static files -# By default, it is stored under /home/inventree/data/static -# Use environment variable INVENTREE_STATIC_ROOT -static_root: '/home/inventree/data/static' +# static_root: '/home/inventree/data/static' + +# Background worker options +background: + workers: 4 + timeout: 90 # Optional URL schemes to allow in URL fields # By default, only the following schemes are allowed: ['http', 'https', 'ftp', 'ftps'] @@ -156,25 +145,14 @@ static_root: '/home/inventree/data/static' # - ssh # Login configuration -# How long do confirmation mail last? -# Use environment variable INVENTREE_LOGIN_CONFIRM_DAYS -#login_confirm_days: 3 -# How many wrong login attempts are permitted? -# Use environment variable INVENTREE_LOGIN_ATTEMPTS -#login_attempts: 5 +login_confirm_days: 3 +login_attempts: 5 # Remote / proxy login # These settings can introduce security problems if configured incorrectly. Please read # https://docs.djangoproject.com/en/4.0/howto/auth-remote-user/ for more details -# Use environment variable INVENTREE_REMOTE_LOGIN -# remote_login: True -# Use environment variable INVENTREE_REMOTE_LOGIN_HEADER -# remote_login_header: REMOTE_USER - -# Add new user on first startup -#admin_user: admin -#admin_email: info@example.com -#admin_password: inventree +remote_login_enabled: False +remote_login_header: REMOTE_USER # Permit custom authentication backends #authentication_backends: diff --git a/InvenTree/plugin/registry.py b/InvenTree/plugin/registry.py index 0d3ceef480..f60972accb 100644 --- a/InvenTree/plugin/registry.py +++ b/InvenTree/plugin/registry.py @@ -22,6 +22,8 @@ from django.utils.text import slugify from maintenance_mode.core import (get_maintenance_mode, maintenance_mode_on, set_maintenance_mode) +from InvenTree.config import get_setting + from .helpers import (IntegrationPluginError, get_plugins, handle_error, log_error) from .plugin import InvenTreePlugin @@ -199,7 +201,7 @@ class PluginsRegistry: if settings.TESTING: custom_dirs = os.getenv('INVENTREE_PLUGIN_TEST_DIR', None) else: - custom_dirs = os.getenv('INVENTREE_PLUGIN_DIR', None) + custom_dirs = get_setting('INVENTREE_PLUGIN_DIR', 'plugin_dir') # Load from user specified directories (unless in testing mode) dirs.append('plugins') diff --git a/docker/requirements.txt b/docker/requirements.txt index 019fa58ab4..2371292f9d 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,9 +1,10 @@ # Base python requirements for docker containers # Basic package requirements +invoke>=1.4.0 # Invoke build tool +pyyaml>=6.0 setuptools==60.0.5 wheel>=0.37.0 -invoke>=1.4.0 # Invoke build tool # Database links psycopg2>=2.9.1 diff --git a/tasks.py b/tasks.py index 5f03c2aade..86041f63e3 100644 --- a/tasks.py +++ b/tasks.py @@ -4,6 +4,7 @@ import json import os import pathlib import re +import shutil import sys from pathlib import Path @@ -522,6 +523,8 @@ def test(c, database=None): def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset"): """Setup a testing enviroment.""" + from InvenTree.InvenTree.config import get_media_dir + if not ignore_update: update(c) @@ -540,8 +543,16 @@ def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset") migrate(c) # Load data - print("Loading data ...") + print("Loading database records ...") import_records(c, filename=f'{path}/inventree_data.json', clear=True) + + # Copy media files + print("Copying media files ...") + src = Path(path).joinpath('media').resolve() + dst = get_media_dir() + + shutil.copytree(src, dst, dirs_exist_ok=True) + print("Done setting up test enviroment...") print("========================================")