diff --git a/.github/workflows/postgresql.yaml b/.github/workflows/postgresql.yaml index 9a56382c4e..b235767110 100644 --- a/.github/workflows/postgresql.yaml +++ b/.github/workflows/postgresql.yaml @@ -27,6 +27,7 @@ jobs: INVENTREE_DEBUG: info INVENTREE_MEDIA_ROOT: ./media INVENTREE_STATIC_ROOT: ./static + INVENTREE_CACHE_HOST: localhost services: postgres: @@ -37,6 +38,11 @@ jobs: ports: - 5432:5432 + redis: + image: redis + ports: + - 6379:6379 + steps: - name: Checkout Code uses: actions/checkout@v2 @@ -49,6 +55,7 @@ jobs: sudo apt-get install libpq-dev pip3 install invoke pip3 install psycopg2 + pip3 install django-redis>=5.0.0 invoke install - name: Run Tests run: invoke test diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 944736e61d..b425a1c41e 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -15,6 +15,7 @@ import logging import os import random +import socket import string import shutil import sys @@ -361,30 +362,6 @@ REST_FRAMEWORK = { WSGI_APPLICATION = 'InvenTree.wsgi.application' -background_workers = os.environ.get('INVENTREE_BACKGROUND_WORKERS', None) - -if background_workers is not None: - try: - background_workers = int(background_workers) - except ValueError: - background_workers = None - -if background_workers is None: - # Sensible default? - background_workers = 4 - -# django-q configuration -Q_CLUSTER = { - 'name': 'InvenTree', - 'workers': background_workers, - 'timeout': 90, - 'retry': 120, - 'queue_limit': 50, - 'bulk': 10, - 'orm': 'default', - 'sync': False, -} - """ Configure the database backend based on the user-specified values. @@ -562,12 +539,84 @@ DATABASES = { } -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, +_cache_config = CONFIG.get("cache", {}) +_cache_host = _cache_config.get("host", os.getenv("INVENTREE_CACHE_HOST")) +_cache_port = _cache_config.get( + "port", os.getenv("INVENTREE_CACHE_PORT", "6379") +) + +if _cache_host: + # We are going to rely upon a possibly non-localhost for our cache, + # so don't wait too long for the cache as nothing in the cache should be + # irreplacable. Django Q Cluster will just try again later. + _cache_options = { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "SOCKET_CONNECT_TIMEOUT": int(os.getenv("CACHE_CONNECT_TIMEOUT", "2")), + "SOCKET_TIMEOUT": int(os.getenv("CACHE_SOCKET_TIMEOUT", "2")), + "CONNECTION_POOL_KWARGS": { + "socket_keepalive": _is_true( + os.getenv("CACHE_TCP_KEEPALIVE", "1") + ), + "socket_keepalive_options": { + socket.TCP_KEEPCNT: int( + os.getenv("CACHE_KEEPALIVES_COUNT", "5") + ), + socket.TCP_KEEPIDLE: int( + os.getenv("CACHE_KEEPALIVES_IDLE", "1") + ), + socket.TCP_KEEPINTVL: int( + os.getenv("CACHE_KEEPALIVES_INTERVAL", "1") + ), + socket.TCP_USER_TIMEOUT: int( + os.getenv("CACHE_TCP_USER_TIMEOUT", "1000") + ), + }, + }, + } + CACHES = { + # Connection configuration for Django Q Cluster + "worker": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://{_cache_host}:{_cache_port}/0", + "OPTIONS": _cache_options, + }, + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://{_cache_host}:{_cache_port}/1", + "OPTIONS": _cache_options, + }, + } +else: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + }, + } + +try: + # 4 background workers seems like a sensible default + background_workers = int(os.environ.get('INVENTREE_BACKGROUND_WORKERS', 4)) +except ValueError: + background_workers = 4 + +# django-q configuration +Q_CLUSTER = { + 'name': 'InvenTree', + 'workers': background_workers, + 'timeout': 90, + 'retry': 120, + 'queue_limit': 50, + 'bulk': 10, + 'orm': 'default', + 'sync': False, } +if _cache_host: + # If using external redis cache, make the cache the broker for Django Q + # as well + Q_CLUSTER["django_redis"] = "worker" + + # Password validation # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators diff --git a/docker/requirements.txt b/docker/requirements.txt index b15d7c538d..1a110d6cde 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -11,3 +11,6 @@ psycopg2>=2.9.1 mysqlclient>=2.0.3 pgcli>=3.1.0 mariadb>=1.0.7 + +# Cache +django-redis>=5.0.0