Merge pull request #2165 from nwns/feature/redisascache

Add support for using Redis as a cache and a broker
This commit is contained in:
Oliver 2021-11-11 17:08:41 +11:00 committed by GitHub
commit b17d6f932a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 87 additions and 28 deletions

View File

@ -27,6 +27,7 @@ jobs:
INVENTREE_DEBUG: info INVENTREE_DEBUG: info
INVENTREE_MEDIA_ROOT: ./media INVENTREE_MEDIA_ROOT: ./media
INVENTREE_STATIC_ROOT: ./static INVENTREE_STATIC_ROOT: ./static
INVENTREE_CACHE_HOST: localhost
services: services:
postgres: postgres:
@ -37,6 +38,11 @@ jobs:
ports: ports:
- 5432:5432 - 5432:5432
redis:
image: redis
ports:
- 6379:6379
steps: steps:
- name: Checkout Code - name: Checkout Code
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -49,6 +55,7 @@ jobs:
sudo apt-get install libpq-dev sudo apt-get install libpq-dev
pip3 install invoke pip3 install invoke
pip3 install psycopg2 pip3 install psycopg2
pip3 install django-redis>=5.0.0
invoke install invoke install
- name: Run Tests - name: Run Tests
run: invoke test run: invoke test

View File

@ -15,6 +15,7 @@ import logging
import os import os
import random import random
import socket
import string import string
import shutil import shutil
import sys import sys
@ -361,30 +362,6 @@ REST_FRAMEWORK = {
WSGI_APPLICATION = 'InvenTree.wsgi.application' WSGI_APPLICATION = 'InvenTree.wsgi.application'
background_workers = os.environ.get('INVENTREE_BACKGROUND_WORKERS', None)
if background_workers is not None:
try:
background_workers = int(background_workers)
except ValueError:
background_workers = None
if background_workers is None:
# Sensible default?
background_workers = 4
# django-q configuration
Q_CLUSTER = {
'name': 'InvenTree',
'workers': background_workers,
'timeout': 90,
'retry': 120,
'queue_limit': 50,
'bulk': 10,
'orm': 'default',
'sync': False,
}
""" """
Configure the database backend based on the user-specified values. Configure the database backend based on the user-specified values.
@ -562,12 +539,84 @@ DATABASES = {
} }
CACHES = { _cache_config = CONFIG.get("cache", {})
'default': { _cache_host = _cache_config.get("host", os.getenv("INVENTREE_CACHE_HOST"))
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', _cache_port = _cache_config.get(
}, "port", os.getenv("INVENTREE_CACHE_PORT", "6379")
)
if _cache_host:
# We are going to rely upon a possibly non-localhost for our cache,
# so don't wait too long for the cache as nothing in the cache should be
# irreplacable. Django Q Cluster will just try again later.
_cache_options = {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"SOCKET_CONNECT_TIMEOUT": int(os.getenv("CACHE_CONNECT_TIMEOUT", "2")),
"SOCKET_TIMEOUT": int(os.getenv("CACHE_SOCKET_TIMEOUT", "2")),
"CONNECTION_POOL_KWARGS": {
"socket_keepalive": _is_true(
os.getenv("CACHE_TCP_KEEPALIVE", "1")
),
"socket_keepalive_options": {
socket.TCP_KEEPCNT: int(
os.getenv("CACHE_KEEPALIVES_COUNT", "5")
),
socket.TCP_KEEPIDLE: int(
os.getenv("CACHE_KEEPALIVES_IDLE", "1")
),
socket.TCP_KEEPINTVL: int(
os.getenv("CACHE_KEEPALIVES_INTERVAL", "1")
),
socket.TCP_USER_TIMEOUT: int(
os.getenv("CACHE_TCP_USER_TIMEOUT", "1000")
),
},
},
}
CACHES = {
# Connection configuration for Django Q Cluster
"worker": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://{_cache_host}:{_cache_port}/0",
"OPTIONS": _cache_options,
},
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://{_cache_host}:{_cache_port}/1",
"OPTIONS": _cache_options,
},
}
else:
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
},
}
try:
# 4 background workers seems like a sensible default
background_workers = int(os.environ.get('INVENTREE_BACKGROUND_WORKERS', 4))
except ValueError:
background_workers = 4
# django-q configuration
Q_CLUSTER = {
'name': 'InvenTree',
'workers': background_workers,
'timeout': 90,
'retry': 120,
'queue_limit': 50,
'bulk': 10,
'orm': 'default',
'sync': False,
} }
if _cache_host:
# If using external redis cache, make the cache the broker for Django Q
# as well
Q_CLUSTER["django_redis"] = "worker"
# Password validation # Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

View File

@ -11,3 +11,6 @@ psycopg2>=2.9.1
mysqlclient>=2.0.3 mysqlclient>=2.0.3
pgcli>=3.1.0 pgcli>=3.1.0
mariadb>=1.0.7 mariadb>=1.0.7
# Cache
django-redis>=5.0.0