From e0e6dbac944ce777ad4437782eb4792742ee9323 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 6 Jan 2022 12:15:33 +1100 Subject: [PATCH 1/5] Add PLUGIN_FILE to settings.py --- InvenTree/InvenTree/settings.py | 7 +++++++ requirements.txt | 1 + 2 files changed, 8 insertions(+) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index ba2808a8dd..e811028ccc 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -89,6 +89,13 @@ with open(cfg_filename, 'r') as cfg: # We will place any config files in the same directory as the config file config_dir = os.path.dirname(cfg_filename) +# Check if the plugin.txt file (specifying required plugins) is specified +PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') + +if not PLUGIN_FILE: + # If not specified, look in the same directory as the configuration file + PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') + # Default action is to run the system in Debug mode # SECURITY WARNING: don't run with debug turned on in production! DEBUG = _is_true(get_setting( diff --git a/requirements.txt b/requirements.txt index d6405bb7bc..a150ae503a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ flake8==3.8.3 # PEP checking gunicorn>=20.1.0 # Gunicorn web server importlib_metadata # Backport for importlib.metadata inventree # Install the latest version of the InvenTree API python library +invoke>=1.6.0 # Invoke management tool (must be installed in venv) markdown==3.3.4 # Force particular version of markdown pep8-naming==0.11.1 # PEP naming convention extension pillow==8.3.2 # Image manipulation From 5c2121b1a115dc83050412f3d5306d4e0c11e0af Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 6 Jan 2022 12:25:07 +1100 Subject: [PATCH 2/5] Add invoke target to install plugins from file --- .gitignore | 1 + InvenTree/InvenTree/settings.py | 22 +++++++++++++++------- tasks.py | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4bc0bb1389..6532442dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ static_i18n # Local config file config.yaml +plugins.txt # Default data file data.json diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index e811028ccc..894332b189 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -89,13 +89,6 @@ with open(cfg_filename, 'r') as cfg: # We will place any config files in the same directory as the config file config_dir = os.path.dirname(cfg_filename) -# Check if the plugin.txt file (specifying required plugins) is specified -PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') - -if not PLUGIN_FILE: - # If not specified, look in the same directory as the configuration file - PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') - # Default action is to run the system in Debug mode # SECURITY WARNING: don't run with debug turned on in production! DEBUG = _is_true(get_setting( @@ -150,6 +143,21 @@ LOGGING = { # Get a logger instance for this setup file logger = logging.getLogger("inventree") +# Check if the plugin.txt file (specifying required plugins) is specified +PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') + +if not PLUGIN_FILE: + # If not specified, look in the same directory as the configuration file + PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') + +if not os.path.exists(PLUGIN_FILE): + logger.warning("Plugin configuration file does not exist") + logger.info(f"Creating plugin file at '{PLUGIN_FILE}'") + + # If opening the file fails (no write permission, for example), then this will throw an error + with open(PLUGIN_FILE, 'w') as plugin_file: + plugin_file.write("# InvenTree Plugins (uses PIP framework to install)\n\n") + """ Specify a secret key to be used by django. diff --git a/tasks.py b/tasks.py index 4d5d7ff6c8..9eaaa261d9 100644 --- a/tasks.py +++ b/tasks.py @@ -78,10 +78,33 @@ def install(c): Installs required python packages """ + print("Installing required python packages from 'requirements.txt'") + # Install required Python packages with PIP c.run('pip3 install -U -r requirements.txt') +@task +def plugins(c): + """ + Installs all plugins as specified in 'plugins.txt' + """ + + try: + from InvenTree.InvenTree.settings import PLUGIN_FILE + except: + print("Error: Could not import PLUGIN_FILE from settings.py") + return + + if not os.path.exists(PLUGIN_FILE): + # Create an empty plugin + print(f"Plugins file '{PLUGIN_FILE}' does not exist") + + print(f"Installing plugin packages from '{PLUGIN_FILE}'") + + # Install the plugins + c.run(f"pip3 install -U -r '{PLUGIN_FILE}'") + @task def shell(c): """ From 75d5f9d6e606fa04e8f78aabab87e277fd90966b Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 6 Jan 2022 12:37:12 +1100 Subject: [PATCH 3/5] Allow specification of an external plugin directory - Specify directory using INVENTREE_PLUGIN_DIR - Specify installation file using INVENTREE_PLUGIN_FILE --- InvenTree/InvenTree/settings.py | 20 ++++++++++++++++++-- docker/Dockerfile | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 894332b189..8fc0562059 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -923,8 +923,7 @@ MARKDOWNIFY_BLEACH = False # Maintenance mode MAINTENANCE_MODE_RETRY_AFTER = 60 - -# Plugins +# Plugin Directories (local plugins will be loaded from these directories) PLUGIN_DIRS = ['plugin.builtin', ] if not TESTING: @@ -935,6 +934,23 @@ if DEBUG or TESTING: # load samples in debug mode PLUGIN_DIRS.append('plugin.samples') +# Check if an external plugin directory has been specified as an environment variable +# Note: This should be specified as INVENTREE_PLUGIN_DIR +plugin_dir = os.getenv('INVENTREE_PLUGIN_DIR') + +if plugin_dir: + if not os.path.exists(plugin_dir): + logger.info(f"Plugin directory '{plugin_dir}' does not exist") + + try: + os.makedirs(plugin_dir, exist_ok=True) + logger.info(f"Created plugin directory '{plugin_dir}'") + except: + logger.warning(f"Could not create plugins directory '{plugin_dir}'") + + if os.path.exists(plugin_dir): + PLUGIN_DIRS.append(plugin_dir) + # Plugin test settings PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING) # are plugins beeing tested? PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False) # load plugins from setup hooks in testing? diff --git a/docker/Dockerfile b/docker/Dockerfile index 673792a22f..8fd932ff6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,9 +30,11 @@ ENV INVENTREE_MNG_DIR="${INVENTREE_HOME}/InvenTree" ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/data" ENV INVENTREE_STATIC_ROOT="${INVENTREE_DATA_DIR}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media" +ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DATA_DIR}/plugins" ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt" +ENV INVENTREE_PLUGIN_FILE="${INVENTREE_DATA_DIR}}/plugins.txt" # Worker configuration (can be altered by user) ENV INVENTREE_GUNICORN_WORKERS="4" From 836b6275b6819e066471bd68871db2965e44df93 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 6 Jan 2022 13:31:04 +1100 Subject: [PATCH 4/5] Removes custom plugins directory - rely on plugins.txt instead --- InvenTree/InvenTree/settings.py | 17 ----------------- docker/Dockerfile | 6 +++++- tasks.py | 25 ++++++++++++------------- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 8fc0562059..d738a640b9 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -934,23 +934,6 @@ if DEBUG or TESTING: # load samples in debug mode PLUGIN_DIRS.append('plugin.samples') -# Check if an external plugin directory has been specified as an environment variable -# Note: This should be specified as INVENTREE_PLUGIN_DIR -plugin_dir = os.getenv('INVENTREE_PLUGIN_DIR') - -if plugin_dir: - if not os.path.exists(plugin_dir): - logger.info(f"Plugin directory '{plugin_dir}' does not exist") - - try: - os.makedirs(plugin_dir, exist_ok=True) - logger.info(f"Created plugin directory '{plugin_dir}'") - except: - logger.warning(f"Could not create plugins directory '{plugin_dir}'") - - if os.path.exists(plugin_dir): - PLUGIN_DIRS.append(plugin_dir) - # Plugin test settings PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING) # are plugins beeing tested? PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False) # load plugins from setup hooks in testing? diff --git a/docker/Dockerfile b/docker/Dockerfile index 8fd932ff6f..55a89210fe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,7 +34,7 @@ ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DATA_DIR}/plugins" ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DATA_DIR}/secret_key.txt" -ENV INVENTREE_PLUGIN_FILE="${INVENTREE_DATA_DIR}}/plugins.txt" +ENV INVENTREE_PLUGIN_FILE="${INVENTREE_DATA_DIR}/plugins.txt" # Worker configuration (can be altered by user) ENV INVENTREE_GUNICORN_WORKERS="4" @@ -131,8 +131,12 @@ ENV INVENTREE_PY_ENV="${INVENTREE_DEV_DIR}/env" # Override default path settings ENV INVENTREE_STATIC_ROOT="${INVENTREE_DEV_DIR}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DEV_DIR}/media" +ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DEV_DIR}/plugins" + ENV INVENTREE_CONFIG_FILE="${INVENTREE_DEV_DIR}/config.yaml" ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DEV_DIR}/secret_key.txt" +ENV INVENTREE_PLUGIN_FILE="${INVENTREE_DEV_DIR}/plugins.txt" + WORKDIR ${INVENTREE_HOME} diff --git a/tasks.py b/tasks.py index 9eaaa261d9..89a579ac9e 100644 --- a/tasks.py +++ b/tasks.py @@ -71,19 +71,6 @@ def manage(c, cmd, pty=False): cmd=cmd ), pty=pty) - -@task -def install(c): - """ - Installs required python packages - """ - - print("Installing required python packages from 'requirements.txt'") - - # Install required Python packages with PIP - c.run('pip3 install -U -r requirements.txt') - - @task def plugins(c): """ @@ -99,12 +86,24 @@ def plugins(c): if not os.path.exists(PLUGIN_FILE): # Create an empty plugin print(f"Plugins file '{PLUGIN_FILE}' does not exist") + return print(f"Installing plugin packages from '{PLUGIN_FILE}'") # Install the plugins c.run(f"pip3 install -U -r '{PLUGIN_FILE}'") +@task(post=[plugins]) +def install(c): + """ + Installs required python packages + """ + + print("Installing required python packages from 'requirements.txt'") + + # Install required Python packages with PIP + c.run('pip3 install -U -r requirements.txt') + @task def shell(c): """ From d8d22e5f38c0451d68b36177a0dda19cbedb30e4 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 6 Jan 2022 14:20:26 +1100 Subject: [PATCH 5/5] Refactor code (so that it actually runs)... - Invoke does not have access to the local virtual environment - Some functions need to be split out from settings.py --- InvenTree/InvenTree/config.py | 90 +++++++++++++++++++++++++++++++++ InvenTree/InvenTree/settings.py | 64 +++-------------------- requirements.txt | 1 - tasks.py | 17 ++----- 4 files changed, 102 insertions(+), 70 deletions(-) create mode 100644 InvenTree/InvenTree/config.py diff --git a/InvenTree/InvenTree/config.py b/InvenTree/InvenTree/config.py new file mode 100644 index 0000000000..35671c1b26 --- /dev/null +++ b/InvenTree/InvenTree/config.py @@ -0,0 +1,90 @@ +""" +Helper functions for loading InvenTree configuration options +""" + +import os +import shutil +import logging + + +logger = logging.getLogger('inventree') + + +def get_base_dir(): + """ Returns the base (top-level) InvenTree directory """ + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def get_config_file(): + """ + Returns the path of the InvenTree configuration file. + + Note: It will be created it if does not already exist! + """ + + base_dir = get_base_dir() + + cfg_filename = os.getenv('INVENTREE_CONFIG_FILE') + + if cfg_filename: + cfg_filename = cfg_filename.strip() + cfg_filename = os.path.abspath(cfg_filename) + else: + # Config file is *not* specified - use the default + cfg_filename = os.path.join(base_dir, 'config.yaml') + + if not os.path.exists(cfg_filename): + print("InvenTree configuration file 'config.yaml' not found - creating default file") + + cfg_template = os.path.join(base_dir, "config_template.yaml") + shutil.copyfile(cfg_template, cfg_filename) + print(f"Created config file {cfg_filename}") + + return cfg_filename + + +def get_plugin_file(): + """ + Returns the path of the InvenTree plugins specification file. + + Note: It will be created if it does not already exist! + """ + # Check if the plugin.txt file (specifying required plugins) is specified + PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') + + if not PLUGIN_FILE: + # If not specified, look in the same directory as the configuration file + + config_dir = os.path.dirname(get_config_file()) + + PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') + + if not os.path.exists(PLUGIN_FILE): + logger.warning("Plugin configuration file does not exist") + logger.info(f"Creating plugin file at '{PLUGIN_FILE}'") + + # If opening the file fails (no write permission, for example), then this will throw an error + with open(PLUGIN_FILE, 'w') as plugin_file: + plugin_file.write("# InvenTree Plugins (uses PIP framework to install)\n\n") + + return PLUGIN_FILE + + +def get_setting(environment_var, backup_val, default_value=None): + """ + Helper function for retrieving a configuration setting value + + - First preference is to look for the environment variable + - Second preference is to look for the value of the settings file + - Third preference is the default value + """ + + val = os.getenv(environment_var) + + if val is not None: + return val + + if backup_val is not None: + return backup_val + + return default_value diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index d738a640b9..89e60a597e 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -17,7 +17,6 @@ import os import random import socket import string -import shutil import sys from datetime import datetime @@ -28,30 +27,12 @@ from django.utils.translation import gettext_lazy as _ from django.contrib.messages import constants as messages import django.conf.locale +from .config import get_base_dir, get_config_file, get_plugin_file, get_setting + def _is_true(x): # Shortcut function to determine if a value "looks" like a boolean - return str(x).lower() in ['1', 'y', 'yes', 't', 'true'] - - -def get_setting(environment_var, backup_val, default_value=None): - """ - Helper function for retrieving a configuration setting value - - - First preference is to look for the environment variable - - Second preference is to look for the value of the settings file - - Third preference is the default value - """ - - val = os.getenv(environment_var) - - if val is not None: - return val - - if backup_val is not None: - return backup_val - - return default_value + return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true'] # Determine if we are running in "test" mode e.g. "manage.py test" @@ -61,27 +42,9 @@ TESTING = 'test' in sys.argv DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = get_base_dir() -# Specify where the "config file" is located. -# By default, this is 'config.yaml' - -cfg_filename = os.getenv('INVENTREE_CONFIG_FILE') - -if cfg_filename: - cfg_filename = cfg_filename.strip() - cfg_filename = os.path.abspath(cfg_filename) - -else: - # Config file is *not* specified - use the default - cfg_filename = os.path.join(BASE_DIR, 'config.yaml') - -if not os.path.exists(cfg_filename): - print("InvenTree configuration file 'config.yaml' not found - creating default file") - - cfg_template = os.path.join(BASE_DIR, "config_template.yaml") - shutil.copyfile(cfg_template, cfg_filename) - print(f"Created config file {cfg_filename}") +cfg_filename = get_config_file() with open(cfg_filename, 'r') as cfg: CONFIG = yaml.safe_load(cfg) @@ -89,6 +52,8 @@ with open(cfg_filename, 'r') as cfg: # We will place any config files in the same directory as the config file config_dir = os.path.dirname(cfg_filename) +PLUGIN_FILE = get_plugin_file() + # Default action is to run the system in Debug mode # SECURITY WARNING: don't run with debug turned on in production! DEBUG = _is_true(get_setting( @@ -143,21 +108,6 @@ LOGGING = { # Get a logger instance for this setup file logger = logging.getLogger("inventree") -# Check if the plugin.txt file (specifying required plugins) is specified -PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE') - -if not PLUGIN_FILE: - # If not specified, look in the same directory as the configuration file - PLUGIN_FILE = os.path.join(config_dir, 'plugins.txt') - -if not os.path.exists(PLUGIN_FILE): - logger.warning("Plugin configuration file does not exist") - logger.info(f"Creating plugin file at '{PLUGIN_FILE}'") - - # If opening the file fails (no write permission, for example), then this will throw an error - with open(PLUGIN_FILE, 'w') as plugin_file: - plugin_file.write("# InvenTree Plugins (uses PIP framework to install)\n\n") - """ Specify a secret key to be used by django. diff --git a/requirements.txt b/requirements.txt index a150ae503a..d6405bb7bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,6 @@ flake8==3.8.3 # PEP checking gunicorn>=20.1.0 # Gunicorn web server importlib_metadata # Backport for importlib.metadata inventree # Install the latest version of the InvenTree API python library -invoke>=1.6.0 # Invoke management tool (must be installed in venv) markdown==3.3.4 # Force particular version of markdown pep8-naming==0.11.1 # PEP naming convention extension pillow==8.3.2 # Image manipulation diff --git a/tasks.py b/tasks.py index 89a579ac9e..34528e2609 100644 --- a/tasks.py +++ b/tasks.py @@ -77,21 +77,14 @@ def plugins(c): Installs all plugins as specified in 'plugins.txt' """ - try: - from InvenTree.InvenTree.settings import PLUGIN_FILE - except: - print("Error: Could not import PLUGIN_FILE from settings.py") - return - - if not os.path.exists(PLUGIN_FILE): - # Create an empty plugin - print(f"Plugins file '{PLUGIN_FILE}' does not exist") - return + from InvenTree.InvenTree.config import get_plugin_file - print(f"Installing plugin packages from '{PLUGIN_FILE}'") + plugin_file = get_plugin_file() + + print(f"Installing plugin packages from '{plugin_file}'") # Install the plugins - c.run(f"pip3 install -U -r '{PLUGIN_FILE}'") + c.run(f"pip3 install -U -r '{plugin_file}'") @task(post=[plugins]) def install(c):