Improve settings.py

- Load database config from either config.yaml or environment variables
- Mix and match, if you want!
- Move to use logging module rather than just printing stuff
- Error if required database parameters are not required

InvenTree/InvenTree/api.py

@@ -5,6 +5,8 @@ Main JSON interface views
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import logging
+
 from django.utils.translation import ugettext as _
 from django.http import JsonResponse
 
@@ -21,7 +23,10 @@ from .version import inventreeVersion, inventreeInstanceName
 from plugins import plugins as inventree_plugins
 
 
-print("Loading action plugins")
+logger = logging.getLogger(__name__)
+
+
+logger.info("Loading action plugins...")
 action_plugins = inventree_plugins.load_action_plugins()
 
 

InvenTree/InvenTree/settings.py

@@ -22,32 +22,58 @@ from datetime import datetime
 from django.utils.translation import gettext_lazy as _
 
 
-def eprint(*args, **kwargs):
-    """ Print a warning message to stderr """
-    print(*args, file=sys.stderr, **kwargs)
-
-
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 cfg_filename = os.path.join(BASE_DIR, 'config.yaml')
 
 if not os.path.exists(cfg_filename):
-    CONFIG = {}
-    eprint("Warning: config.yaml not found - using default settings")
-else:
-    with open(cfg_filename, 'r') as cfg:
+    print("Error: config.yaml not found")
+    sys.exit(-1)
+
+with open(cfg_filename, 'r') as cfg:
     CONFIG = yaml.safe_load(cfg)
 
-# Read the autogenerated key-file
-key_file = open(os.path.join(BASE_DIR, 'secret_key.txt'), 'r')
-
-SECRET_KEY = key_file.read().strip()
-
 # Default action is to run the system in Debug mode
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = CONFIG.get('debug', True)
 
+# Configure logging settings
+
+log_level = CONFIG.get('log_level', 'DEBUG').upper()
+
+if log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
+    log_level = 'WARNING'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': log_level,
+    },
+}
+
+logging.basicConfig(
+    level=log_level,
+    format='%(asctime)s %(levelname)s %(message)s',
+)
+
+# Get a logger instance for this setup file
+logger = logging.getLogger(__name__)
+
+# Read the autogenerated key-file
+key_file_name = os.path.join(BASE_DIR, 'secret_key.txt')
+logger.info(f'Loading SERCRET_KEY from {key_file_name}')
+key_file = open(key_file_name, 'r')
+
+SECRET_KEY = key_file.read().strip()
+
 # List of allowed hosts (default = allow all)
 ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*'])
 
@@ -65,13 +91,6 @@ if cors_opt:
     if not CORS_ORIGIN_ALLOW_ALL:
         CORS_ORIGIN_WHITELIST = cors_opt.get('whitelist', [])
 
-if DEBUG:
-    # will output to your console
-    logging.basicConfig(
-        level=logging.DEBUG,
-        format='%(asctime)s %(levelname)s %(message)s',
-    )
-
 # Web URL endpoint for served static files
 STATIC_URL = '/static/'
 
@@ -92,14 +111,18 @@ MEDIA_URL = '/media/'
 MEDIA_ROOT = os.path.abspath(CONFIG.get('media_root', os.path.join(BASE_DIR, 'media')))
 
 if DEBUG:
-    print("InvenTree running in DEBUG mode")
-    print("MEDIA_ROOT:", MEDIA_ROOT)
-    print("STATIC_ROOT:", STATIC_ROOT)
+    logger.info("InvenTree running in DEBUG mode")
+    
+logger.info(f"MEDIA_ROOT: '{MEDIA_ROOT}'")
+logger.info(f"STATIC_ROOT: '{STATIC_ROOT}'")
 
 # Does the user wish to use the sentry.io integration?
 sentry_opts = CONFIG.get('sentry', {})
 
 if sentry_opts.get('enabled', False):
+
+    logger.info("Configuring sentry.io integration")
+
     dsn = sentry_opts.get('dsn', None)
 
     if dsn is not None:
@@ -111,11 +134,11 @@ if sentry_opts.get('enabled', False):
             sentry_sdk.init(dsn=dsn, integrations=[DjangoIntegration()], send_default_pii=True)
 
         except ModuleNotFoundError:
-            print("sentry_sdk module not found. Install using 'pip install sentry-sdk'")
+            logger.error("sentry_sdk module not found. Install using 'pip install sentry-sdk'")
             sys.exit(-1)
 
     else:
-        print("Warning: Sentry.io DSN not specified")
+        logger.warning("Sentry.io DSN not specified in config file")
 
 # Application definition
 
@@ -160,17 +183,6 @@ INSTALLED_APPS = [
     'error_report',                 # Error reporting in the admin interface
 ]
 
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-        },
-    },
-}
-
 MIDDLEWARE = CONFIG.get('middleware', [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
@@ -193,7 +205,7 @@ AUTHENTICATION_BACKENDS = CONFIG.get('authentication_backends', [
 
 # If the debug toolbar is enabled, add the modules
 if DEBUG and CONFIG.get('debug_toolbar', False):
-    print("Running with DEBUG_TOOLBAR enabled")
+    logger.info("Running with DEBUG_TOOLBAR enabled")
     INSTALLED_APPS.append('debug_toolbar')
     MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
 
@@ -285,7 +297,7 @@ When running unit tests, enforce usage of sqlite3 database,
 so that the tests can be run in RAM without any setup requirements
 """
 if 'test' in sys.argv:
-    eprint('InvenTree: Running tests - Using sqlite3 memory database')
+    logger.info('InvenTree: Running tests - Using sqlite3 memory database')
     DATABASES['default'] = {
         # Ensure sqlite3 backend is being used
         'ENGINE': 'django.db.backends.sqlite3',
@@ -295,14 +307,69 @@ if 'test' in sys.argv:
 
 # Database backend selection
 else:
-    if 'database' in CONFIG:
-        DATABASES['default'] = CONFIG['database']
+    """
+    Configure the database backend based on the user-specified values.
+    
+    - Primarily this configuration happens in the config.yaml file
+    - However there may be reason to configure the DB via environmental variables
+    - The following code lets the user "mix and match" database configuration
+    """
+    
+    logger.info("Configuring database backend:")
+
+    # Extract database configuration from the config.yaml file
+    db_config = CONFIG.get('database', {})
+
+    # If a particular database option is not specified in the config file,
+    # look for it in the environmental variables
+    # e.g. INVENTREE_DB_NAME / INVENTREE_DB_USER / etc
+
+    db_keys = ['ENGINE', 'NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']
+
+    for key in db_keys:
+        if key not in db_config:
+            logger.debug(f" - Missing {key} value: Looking for environment variable INVENTREE_DB_{key}")
+            env_key = f'INVENTREE_DB_{key}'
+            env_var = os.environ.get(env_key, None)
+
+            if env_var is not None:
+                logger.info(f'Using environment variable INVENTREE_DB_{key}')
+                db_config[key] = env_var
             else:
-        eprint("Warning: Database backend not specified - using default (sqlite)")
-        DATABASES['default'] = {
-            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': os.path.join(BASE_DIR, 'inventree_db.sqlite3'),
-        }
+                logger.debug(f'    INVENTREE_DB_{key} not found in environment variables')
+
+    # Check that required database configuration options are specified
+    reqiured_keys = ['ENGINE', 'NAME']
+    
+    for key in reqiured_keys:
+        if key not in db_config:
+            error_msg = f'Missing required database configuration value {key} in config.yaml'
+            logger.error(error_msg)
+
+            print('Error: ' + error_msg)
+            sys.exit(-1)
+
+    """
+    Special considerations for the database 'ENGINE' setting.
+    It can be specified in config.yaml (or envvar) as either (for example):
+    - sqlite3
+    - django.db.backends.sqlite3
+    - django.db.backends.postgresql
+    """
+
+    db_engine = db_config['ENGINE']
+
+    if db_engine.lower() in ['sqlite3', 'postgresql', 'mysql']:
+        # Prepend the required python module string
+        db_engine = f'django.db.backends.{db_engine.lower()}'
+        db_config['ENGINE'] = db_engine
+
+    db_name = db_config['NAME']
+
+    logger.info(f"Database ENGINE: '{db_engine}'")
+    logger.info(f"Database NAME: '{db_name}'")
+
+    DATABASES['default'] = db_config
 
 CACHES = {
     'default': {
@@ -341,7 +408,7 @@ AUTH_PASSWORD_VALIDATORS = [
 EXTRA_URL_SCHEMES = CONFIG.get('extra_url_schemes', [])
 
 if not type(EXTRA_URL_SCHEMES) in [list]:
-    eprint("Warning: extra_url_schemes not correctly formatted")
+    logger.warning("extra_url_schemes not correctly formatted")
     EXTRA_URL_SCHEMES = []
 
 # Internationalization

InvenTree/barcode/barcode.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import hashlib
+import logging
 
 from InvenTree import plugins as InvenTreePlugins
 from barcode import plugins as BarcodePlugins
@@ -10,6 +11,9 @@ from stock.serializers import StockItemSerializer, LocationSerializer
 from part.serializers import PartSerializer
 
 
+logger = logging.getLogger(__name__)
+
+
 def hash_barcode(barcode_data):
     """
     Calculate an MD5 hash of barcode data
@@ -130,18 +134,17 @@ def load_barcode_plugins(debug=False):
     Function to load all barcode plugins
     """
 
-    if debug:
-        print("Loading barcode plugins")
+    logger.debug("Loading barcode plugins")
 
     plugins = InvenTreePlugins.get_plugins(BarcodePlugins, BarcodePlugin)
 
     if debug:
         if len(plugins) > 0:
-            print("Discovered {n} plugins:".format(n=len(plugins)))
+            logger.info(f"Discovered {len(plugins)} barcode plugins")
 
             for p in plugins:
-                print(" - {p}".format(p=p.PLUGIN_NAME))
+                logger.debug(" - {p}".format(p=p.PLUGIN_NAME))
         else:
-            print("No barcode plugins found")
+            logger.debug("No barcode plugins found")
 
     return plugins

InvenTree/company/apps.py

@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
 from django.apps import AppConfig
 from django.db.utils import OperationalError, ProgrammingError
 from django.conf import settings
 
 
+logger = logging.getLogger(__name__)
+
+
 class CompanyConfig(AppConfig):
     name = 'company'
 
@@ -21,7 +25,7 @@ class CompanyConfig(AppConfig):
 
         from .models import Company
 
-        print("InvenTree: Checking Company image thumbnails")
+        logger.debug("Checking Company image thumbnails")
 
         try:
             for company in Company.objects.all():
@@ -30,11 +34,11 @@ class CompanyConfig(AppConfig):
                     loc = os.path.join(settings.MEDIA_ROOT, url)
 
                     if not os.path.exists(loc):
-                        print("InvenTree: Generating thumbnail for Company '{c}'".format(c=company.name))
+                        logger.info("InvenTree: Generating thumbnail for Company '{c}'".format(c=company.name))
                         try:
                             company.image.render_variations(replace=False)
                         except FileNotFoundError:
-                            print("Image file missing")
+                            logger.warning("Image file missing")
                             company.image = None
                             company.save()
         except (OperationalError, ProgrammingError):

InvenTree/config_template.yaml

@@ -1,22 +1,41 @@
 
 # Database backend selection - Configure backend database settings
-# Ref: https://docs.djangoproject.com/en/2.2/ref/settings/#std:setting-DATABASES
+# Ref: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-DATABASES
 # Specify database parameters below as they appear in the Django docs
+
+# Note: Database configuration options can also be specified from environmental variables,
+#       with the prefix INVENTREE_DB_
+#       e.g INVENTREE_DB_NAME / INVENTREE_DB_USER / INVENTREE_DB_PASSWORD
 database:
-  # Example configuration - sqlite (default)
-  ENGINE: django.db.backends.sqlite3
+  # Default configuration - sqlite filesystem database
+  ENGINE: sqlite3
   NAME: '../inventree_default_db.sqlite3' 
   
   # For more complex database installations, further parameters are required
   # Refer to the django documentation for full list of options
   
-  # Example Configuration - MySQL
+  # --- Available options: ---
+  # ENGINE: Database engine. Selection from:
+  #         - sqlite3
+  #         - mysql
+  #         - postgresql
+  # NAME: Database name
+  # USER: Database username (if required)
+  # PASSWORD: Database password (if required)
+  # HOST: Database host address (if required)
+  # PORT: Database host port (if required)
+
+  # --- Example Configuration - sqlite3 ---
+  # ENGINE: sqlite3
+  # NAME: '/path/to/database.sqlite3'
+
+  # --- Example Configuration - MySQL ---
   #ENGINE: django.db.backends.mysql
   #NAME: inventree
   #USER: inventree_username
   #PASSWORD: inventree_password
-  #HOST: ''
-  #PORT: ''
+  #HOST: '127.0.0.1'
+  #PORT: '5432'
 
 # Select default system language (default is 'en-us')
 language: en-us
@@ -45,6 +64,9 @@ debug: True
 #       and only if InvenTree is accessed from a local IP (127.0.0.1)
 debug_toolbar: False
 
+# Configure the system logging level
+# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL
+log_level: WARNING
 
 # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation)
 # A list of strings representing the host/domain names that this Django site can serve.

InvenTree/part/apps.py

@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
 from django.db.utils import OperationalError, ProgrammingError
 from django.apps import AppConfig
 from django.conf import settings
 
 
+logger = logging.getLogger(__name__)
+
+
 class PartConfig(AppConfig):
     name = 'part'
 
@@ -27,7 +31,7 @@ class PartConfig(AppConfig):
 
         from .models import Part
 
-        print("InvenTree: Checking Part image thumbnails")
+        logger.debug("InvenTree: Checking Part image thumbnails")
 
         try:
             for part in Part.objects.all():
@@ -36,11 +40,11 @@ class PartConfig(AppConfig):
                     loc = os.path.join(settings.MEDIA_ROOT, url)
                     
                     if not os.path.exists(loc):
-                        print("InvenTree: Generating thumbnail for Part '{p}'".format(p=part.name))
+                        logger.info("InvenTree: Generating thumbnail for Part '{p}'".format(p=part.name))
                         try:
                             part.image.render_variations(replace=False)
                         except FileNotFoundError:
-                            print("Image file missing")
+                            logger.warning("Image file missing")
                             part.image = None
                             part.save()
         except (OperationalError, ProgrammingError):

InvenTree/part/views.py

@@ -1320,8 +1320,6 @@ class BomUpload(InvenTreeRoleMixin, FormView):
                 except KeyError:
                     pass
 
-            print(row, row['part_match'], len(row['part_options']))
-
     def extractDataFromFile(self, bom):
         """ Read data from the BOM file """
 

InvenTree/plugins/action/action.py

@@ -1,8 +1,13 @@
 # -*- coding: utf-8 -*-
 
+import logging
+
 import plugins.plugin as plugin
 
 
+logger = logging.getLogger(__name__)
+
+
 class ActionPlugin(plugin.InvenTreePlugin):
     """
     The ActionPlugin class is used to perform custom actions

InvenTree/plugins/plugins.py

@@ -3,12 +3,16 @@
 import inspect
 import importlib
 import pkgutil
+import logging
 
 # Action plugins
 import plugins.action as action
 from plugins.action.action import ActionPlugin
 
 
+logger = logging.getLogger(__name__)
+
+
 def iter_namespace(pkg):
 
     return pkgutil.iter_modules(pkg.__path__, pkg.__name__ + ".")
@@ -52,14 +56,14 @@ def load_action_plugins():
     Return a list of all registered action plugins
     """
 
-    print("Loading action plugins")
+    logger.debug("Loading action plugins")
 
     plugins = get_plugins(action, ActionPlugin)
 
     if len(plugins) > 0:
-        print("Discovered {n} action plugins:".format(n=len(plugins)))
+        logger.info("Discovered {n} action plugins:".format(n=len(plugins)))
 
         for ap in plugins:
-            print(" - {ap}".format(ap=ap.PLUGIN_NAME))
+            logger.debug(" - {ap}".format(ap=ap.PLUGIN_NAME))
 
     return plugins