diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index be06bb05c7..4254dd661c 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -26,6 +26,7 @@ from dotenv import load_dotenv from InvenTree.config import get_boolean_setting, get_custom_file, get_setting from InvenTree.sentry import default_sentry_dsn, init_sentry +from InvenTree.tracing import setup_instruments, setup_tracing from InvenTree.version import checkMinPythonVersion, inventreeApiVersion from . import config, locales @@ -711,6 +712,14 @@ REMOTE_LOGIN_HEADER = get_setting( 'INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER' ) +# region Tracing / error tracking +inventree_tags = { + 'testing': TESTING, + 'docker': DOCKER, + 'debug': DEBUG, + 'remote': REMOTE_LOGIN, +} + # sentry.io integration for error reporting SENTRY_ENABLED = get_boolean_setting( 'INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False @@ -723,15 +732,42 @@ SENTRY_SAMPLE_RATE = float( ) if SENTRY_ENABLED and SENTRY_DSN: # pragma: no cover - inventree_tags = { - 'testing': TESTING, - 'docker': DOCKER, - 'debug': DEBUG, - 'remote': REMOTE_LOGIN, - } - init_sentry(SENTRY_DSN, SENTRY_SAMPLE_RATE, inventree_tags) +# OpenTelemetry tracing +TRACING_ENABLED = get_boolean_setting( + 'INVENTREE_TRACING_ENABLED', 'tracing.enabled', False +) +if TRACING_ENABLED: # pragma: no cover + _t_endpoint = get_setting('INVENTREE_TRACING_ENDPOINT', 'tracing.endpoint', None) + _t_headers = get_setting('INVENTREE_TRACING_HEADERS', 'tracing.headers', None, dict) + if _t_endpoint and _t_headers: + _t_resources = get_setting( + 'INVENTREE_TRACING_RESOURCES', 'tracing.resources', {}, dict + ) + cstm_tags = {'inventree.env.' + k: v for k, v in inventree_tags.items()} + tracing_resources = {**cstm_tags, **_t_resources} + + setup_tracing( + _t_endpoint, + _t_headers, + tracing_resources, + get_boolean_setting('INVENTREE_TRACING_CONSOLE', 'tracing.console', False), + get_setting('INVENTREE_TRACING_AUTH', 'tracing.auth', {}), + get_setting('INVENTREE_TRACING_IS_HTTP', 'tracing.is_http', False), + get_boolean_setting( + 'INVENTREE_TRACING_APPEND_HTTP', 'tracing.append_http', True + ), + ) + # Run tracing/logging instrumentation + setup_instruments() + else: + logger.warning( + 'OpenTelemetry tracing not enabled because endpoint or headers are not set' + ) + +# endregion + # Cache configuration cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None) cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379', typecast=int) diff --git a/InvenTree/InvenTree/tracing.py b/InvenTree/InvenTree/tracing.py new file mode 100644 index 0000000000..b14dcc4a0d --- /dev/null +++ b/InvenTree/InvenTree/tracing.py @@ -0,0 +1,142 @@ +"""OpenTelemetry setup functions.""" + +import base64 +import logging +from typing import Optional + +from opentelemetry import metrics, trace +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.redis import RedisInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.sdk import _logs as logs +from opentelemetry.sdk import resources +from opentelemetry.sdk._logs import export as logs_export +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, + PeriodicExportingMetricReader, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + +from InvenTree.version import inventreeVersion + +# Logger configuration +logger = logging.getLogger('inventree') + + +def setup_tracing( + endpoint: str, + headers: dict, + resources_input: Optional[dict] = None, + console: bool = False, + auth: Optional[dict] = None, + is_http: bool = False, + append_http: bool = True, +): + """Set up tracing for the application in the current context. + + Args: + endpoint: The endpoint to send the traces to. + headers: The headers to send with the traces. + resources_input: The resources to send with the traces. + console: Whether to output the traces to the console. + """ + if resources_input is None: + resources_input = {} + if auth is None: + auth = {} + + # Setup the auth headers + if 'basic' in auth: + basic_auth = auth['basic'] + if 'username' in basic_auth and 'password' in basic_auth: + auth_raw = f'{basic_auth["username"]}:{basic_auth["password"]}' + auth_token = base64.b64encode(auth_raw.encode('utf-8')).decode('utf-8') + headers['Authorization'] = f'Basic {auth_token}' + else: + logger.warning('Basic auth is missing username or password') + + # Clean up headers + headers = {k: v for k, v in headers.items() if v is not None} + + # Initialize the OTLP Resource + resource = resources.Resource( + attributes={ + resources.SERVICE_NAME: 'BACKEND', + resources.SERVICE_NAMESPACE: 'INVENTREE', + resources.SERVICE_VERSION: inventreeVersion(), + **resources_input, + } + ) + + # Import the OTLP exporters + if is_http: + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + else: + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + # Spans / Tracs + span_exporter = OTLPSpanExporter( + headers=headers, + endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/traces', + ) + trace_processor = BatchSpanProcessor(span_exporter) + trace_provider = TracerProvider(resource=resource) + trace.set_tracer_provider(trace_provider) + trace_provider.add_span_processor(trace_processor) + # For debugging purposes, export the traces to the console + if console: + trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + + # Metrics + metric_perodic_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + headers=headers, + endpoint=endpoint + if not (is_http and append_http) + else f'{endpoint}/v1/metrics', + ) + ) + metric_readers = [metric_perodic_reader] + + # For debugging purposes, export the metrics to the console + if console: + console_metric_exporter = ConsoleMetricExporter() + console_metric_reader = PeriodicExportingMetricReader(console_metric_exporter) + metric_readers.append(console_metric_reader) + + meter_provider = MeterProvider(resource=resource, metric_readers=metric_readers) + metrics.set_meter_provider(meter_provider) + + # Logs + log_exporter = OTLPLogExporter( + headers=headers, + endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/logs', + ) + log_provider = logs.LoggerProvider(resource=resource) + log_provider.add_log_record_processor( + logs_export.BatchLogRecordProcessor(log_exporter) + ) + handler = logs.LoggingHandler(level=logging.INFO, logger_provider=log_provider) + logger = logging.getLogger('inventree') + logger.addHandler(handler) + + +def setup_instruments(): + """Run auto-insturmentation for OpenTelemetry tracing.""" + DjangoInstrumentor().instrument() + RedisInstrumentor().instrument() + RequestsInstrumentor().instrument() diff --git a/InvenTree/config_template.yaml b/InvenTree/config_template.yaml index a6c366280d..9baef5fbce 100644 --- a/InvenTree/config_template.yaml +++ b/InvenTree/config_template.yaml @@ -136,6 +136,25 @@ sentry_enabled: False #sentry_sample_rate: 0.1 #sentry_dsn: https://custom@custom.ingest.sentry.io/custom +# OpenTelemetry tracing/metrics - disabled by default +# This can be used to send tracing data, logs and metrics to OpenTelemtry compatible backends +# See https://opentelemetry.io/ecosystem/vendors/ for a list of supported backends +# Alternatively, use environment variables eg. INVENTREE_TRACING_ENABLED, INVENTREE_TRACING_HEADERS, INVENTREE_TRACING_AUTH +#tracing: +# enabled: true +# endpoint: https://otlp-gateway-prod-eu-west-0.grafana.net/otlp +# headers: +# api-key: 'sample' +# auth: +# basic: +# username: '******' +# password: 'glc_****' +# is_http: true +# append_http: true +# console: false +# resources: +# CUSTOM_KEY: 'CUSTOM_VALUE' + # Set this variable to True to enable InvenTree Plugins # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED plugins_enabled: False diff --git a/InvenTree/manage.py b/InvenTree/manage.py index c3e3fb6be6..bc489b085a 100755 --- a/InvenTree/manage.py +++ b/InvenTree/manage.py @@ -8,6 +8,7 @@ import sys def main(): """Run administrative tasks.""" os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings') + try: from django.core.management import execute_from_command_line except ImportError as exc: # pragma: no cover diff --git a/docs/docs/start/advanced.md b/docs/docs/start/advanced.md index 7936fc3559..1064e318de 100644 --- a/docs/docs/start/advanced.md +++ b/docs/docs/start/advanced.md @@ -71,3 +71,20 @@ Next you can start configuring the connection. Either use the config file or set | `ldap.require_group` | `INVENTREE_LDAP_REQUIRE_GROUP` | If set, users _must_ be in this group to log in to InvenTree | | `ldap.deny_group` | `INVENTREE_LDAP_DENY_GROUP` | If set, users _must not_ be in this group to log in to InvenTree | | `ldap.user_flags_by_group` | `INVENTREE_LDAP_USER_FLAGS_BY_GROUP` | LDAP group to InvenTree user flag map, can be json if used as env, in yml directly specify the object. See config template for example, default: `{}` | + + +## Tracing support + +Starting with 0.14.0 InvenTree supports sending traces, logs and metrics to OpenTelemetry compatible endpoints (both HTTP and gRPC). A [list of vendors](https://opentelemetry.io/ecosystem/vendors) is available on the project site. +This can be used to track usage and performance of the InvenTree backend and connected services like databases, caches and more. + +| config key | ENV Variable | Description | +| --- | --- | --- | +| `tracing.enabled` | `INVENTREE_TRACING_ENABLED` | Set this to `True` to enable OpenTelemetry. | +| `tracing.endpoint` | `INVENTREE_TRACING_ENDPOINT` | General endpoint for information (not specific trace/log url) | +| `tracing.headers` | `INVENTREE_TRACING_HEADERS` | HTTP headers that should be send with every request (often used for authentication). Format as a dict. | +| `tracing.auth.basic` | `INVENTREE_TRACING_AUTH_BASIC` | Auth headers that should be send with every requests (will be encoded to b64 and overwrite auth headers) | +| `tracing.is_http` | `INVENTREE_TRACING_IS_HTTP` | Are the endpoints HTTP (True, default) or gRPC (False) | +| `tracing.append_http` | `INVENTREE_TRACING_APPEND_HTTP` | Append default url routes (v1) to `tracing.endpoint` | +| `tracing.console` | `INVENTREE_TRACING_CONSOLE` | Print out all exports (additionally) to the console for debugging. Do not use in production | +| `tracing.resources` | `INVENTREE_TRACING_RESOURCES` | Add additional resources to all exports. This can be used to add custom tags to the traces. Format as a dict. | diff --git a/requirements.in b/requirements.in index 77bfe14863..ce2a2a36bb 100644 --- a/requirements.in +++ b/requirements.in @@ -49,3 +49,11 @@ sentry-sdk # Error reporting (optional) setuptools # Standard dependency tablib[xls,xlsx,yaml] # Support for XLS and XLSX formats weasyprint # PDF generation + +# OpenTelemetry dependencies +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +opentelemetry-instrumentation-django +opentelemetry-instrumentation-requests +opentelemetry-instrumentation-redis diff --git a/requirements.txt b/requirements.txt index ef3f9d6b95..22652b35c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,11 @@ attrs==23.1.0 # referencing babel==2.13.1 # via py-moneyed +backoff==2.2.1 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http bleach[css]==6.1.0 # via # bleach @@ -45,6 +50,11 @@ defusedxml==0.7.1 # via # odfpy # python3-openid +deprecated==1.2.14 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http diff-match-patch==20230430 # via django-import-export dj-rest-auth==5.0.2 @@ -167,6 +177,12 @@ fonttools[woff]==4.44.0 # via # fonttools # weasyprint +googleapis-common-protos==1.62.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.60.0 + # via opentelemetry-exporter-otlp-proto-grpc gunicorn==21.2.0 # via -r requirements.in html5lib==1.1 @@ -179,6 +195,7 @@ importlib-metadata==6.8.0 # via # django-q2 # markdown + # opentelemetry-api inflection==0.5.1 # via drf-spectacular itypes==1.2.0 @@ -201,6 +218,63 @@ odfpy==1.4.1 # via tablib openpyxl==3.1.2 # via tablib +opentelemetry-api==1.22.0 + # via + # -r requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-django + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-instrumentation-wsgi + # opentelemetry-sdk +opentelemetry-exporter-otlp==1.22.0 + # via -r requirements.in +opentelemetry-exporter-otlp-proto-common==1.22.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.22.0 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.22.0 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.43b0 + # via + # opentelemetry-instrumentation-django + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-instrumentation-wsgi +opentelemetry-instrumentation-django==0.43b0 + # via -r requirements.in +opentelemetry-instrumentation-redis==0.43b0 + # via -r requirements.in +opentelemetry-instrumentation-requests==0.43b0 + # via -r requirements.in +opentelemetry-instrumentation-wsgi==0.43b0 + # via opentelemetry-instrumentation-django +opentelemetry-proto==1.22.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.22.0 + # via + # -r requirements.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.43b0 + # via + # opentelemetry-instrumentation-django + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-instrumentation-wsgi + # opentelemetry-sdk +opentelemetry-util-http==0.43b0 + # via + # opentelemetry-instrumentation-django + # opentelemetry-instrumentation-requests + # opentelemetry-instrumentation-wsgi packaging==23.2 # via gunicorn pdf2image==1.16.3 @@ -215,6 +289,10 @@ pillow==10.1.0 # weasyprint pint==0.21 # via -r requirements.in +protobuf==4.25.2 + # via + # googleapis-common-protos + # opentelemetry-proto py-moneyed==3.0 # via django-money pycparser==2.21 @@ -271,6 +349,7 @@ requests==2.31.0 # via # coreapi # django-allauth + # opentelemetry-exporter-otlp-proto-http # requests-oauthlib requests-oauthlib==1.3.1 # via django-allauth @@ -305,6 +384,7 @@ tinycss2==1.2.1 typing-extensions==4.8.0 # via # asgiref + # opentelemetry-sdk # py-moneyed # qrcode uritemplate==4.1.1 @@ -326,6 +406,11 @@ webencodings==0.5.1 # cssselect2 # html5lib # tinycss2 +wrapt==1.16.0 + # via + # deprecated + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis xlrd==2.0.1 # via tablib xlwt==1.3.0