Add a "heartbeat" task which runs every 5 minutes

- Allows us to track if the worker is running
- Due to Stat.get_all() not always working
This commit is contained in:
Oliver Walters 2021-03-12 15:27:28 +11:00
parent 4925f24ca9
commit bfb0cb3b47
4 changed files with 45 additions and 9 deletions

View File

@ -1,8 +1,10 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from django.apps import AppConfig
import logging import logging
from django.apps import AppConfig
from django.core.exceptions import AppRegistryNotReady
import InvenTree.tasks import InvenTree.tasks
@ -34,3 +36,9 @@ class InvenTreeConfig(AppConfig):
'InvenTree.tasks.check_for_updates', 'InvenTree.tasks.check_for_updates',
schedule_type=Schedule.DAILY schedule_type=Schedule.DAILY
) )
InvenTree.tasks.schedule_task(
'InvenTree.tasks.heartbeat',
schedule_type=Schedule.MINUTES,
minutes=5
)

View File

@ -6,9 +6,11 @@ Provides system status functionality checks.
from __future__ import unicode_literals from __future__ import unicode_literals
import logging import logging
from datetime import datetime, timedelta
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
from django_q.models import Success
from django_q.monitor import Stat from django_q.monitor import Stat
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -21,10 +23,25 @@ def is_q_cluster_running(**kwargs):
clusters = Stat.get_all() clusters = Stat.get_all()
for cluster in clusters: if len(clusters) > 0:
print("Cluster:", cluster) return True
return len(clusters) > 0 """
Sometimes Stat.get_all() returns [].
In this case we have the 'heartbeat' task running every five minutes.
Check to see if we have a result within the last ten minutes
"""
now = datetime.now()
past = now - timedelta(minutes=10)
results = Success.objects.filter(
func='InvenTree.tasks.heartbeat',
started__gte=past
)
# If any results are returned, then the background worker is running!
return results.exists()
def check_system_health(**kwargs): def check_system_health(**kwargs):

View File

@ -6,8 +6,6 @@ import json
import requests import requests
import logging import logging
from datetime import timedelta
from django.core.exceptions import AppRegistryNotReady from django.core.exceptions import AppRegistryNotReady
@ -37,6 +35,17 @@ def schedule_task(taskname, **kwargs):
) )
def heartbeat():
"""
Simple task which runs at 5 minute intervals,
so we can determine that the background worker
is actually running.
(There is probably a less "hacky" way of achieving this)
"""
pass
def delete_successful_tasks(): def delete_successful_tasks():
""" """
Delete successful task logs Delete successful task logs
@ -45,6 +54,7 @@ def delete_successful_tasks():
pass pass
def check_for_updates(): def check_for_updates():
""" """
Check if there is an update for InvenTree Check if there is an update for InvenTree
@ -52,7 +62,6 @@ def check_for_updates():
try: try:
import common.models import common.models
import InvenTree.version
except AppRegistryNotReady: except AppRegistryNotReady:
return return
@ -67,7 +76,7 @@ def check_for_updates():
tag = data.get('tag_name', None) tag = data.get('tag_name', None)
if not tag: if not tag:
logger.warning(f"'tag_name' missing from GitHub response") logger.warning("'tag_name' missing from GitHub response")
return return
match = re.match(r"^.*(\d+)\.(\d+)\.(\d+).*$", tag) match = re.match(r"^.*(\d+)\.(\d+)\.(\d+).*$", tag)

View File

@ -59,8 +59,10 @@
{% endif %} {% endif %}
<li class='dropdown'> <li class='dropdown'>
<a class='dropdown-toggle' data-toggle='dropdown' href="#"> <a class='dropdown-toggle' data-toggle='dropdown' href="#">
{% if not system_healthy or not up_to_date %} {% if not system_healthy %}
<span class='fas fa-exclamation-triangle icon-red'></span> <span class='fas fa-exclamation-triangle icon-red'></span>
{% elif not up_to_date %}
<span class='fas fa-info-circle icon-green'></span>
{% endif %} {% endif %}
<span class="fas fa-user"></span> <b>{{ user.get_username }}</b></a> <span class="fas fa-user"></span> <b>{{ user.get_username }}</b></a>
<ul class='dropdown-menu'> <ul class='dropdown-menu'>