From bfb0cb3b4754109657fb29c353d450f4de94f5dd Mon Sep 17 00:00:00 2001
From: Oliver Walters <oliver.henry.walters@gmail.com>
Date: Fri, 12 Mar 2021 15:27:28 +1100
Subject: [PATCH] Add a "heartbeat" task which runs every 5 minutes

- Allows us to track if the worker is running
- Due to Stat.get_all() not always working
---
 InvenTree/InvenTree/apps.py     | 10 +++++++++-
 InvenTree/InvenTree/status.py   | 23 ++++++++++++++++++++---
 InvenTree/InvenTree/tasks.py    | 17 +++++++++++++----
 InvenTree/templates/navbar.html |  4 +++-
 4 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py
index 0b5283706c..902ef2c648 100644
--- a/InvenTree/InvenTree/apps.py
+++ b/InvenTree/InvenTree/apps.py
@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
 
-from django.apps import AppConfig
 import logging
 
+from django.apps import AppConfig
+from django.core.exceptions import AppRegistryNotReady
+
 import InvenTree.tasks
 
 
@@ -34,3 +36,9 @@ class InvenTreeConfig(AppConfig):
             'InvenTree.tasks.check_for_updates',
             schedule_type=Schedule.DAILY
         )
+
+        InvenTree.tasks.schedule_task(
+            'InvenTree.tasks.heartbeat',
+            schedule_type=Schedule.MINUTES,
+            minutes=5
+        )
\ No newline at end of file
diff --git a/InvenTree/InvenTree/status.py b/InvenTree/InvenTree/status.py
index e02e476a51..1f6b01053c 100644
--- a/InvenTree/InvenTree/status.py
+++ b/InvenTree/InvenTree/status.py
@@ -6,9 +6,11 @@ Provides system status functionality checks.
 from __future__ import unicode_literals
 
 import logging
+from datetime import datetime, timedelta
 
 from django.utils.translation import ugettext as _
 
+from django_q.models import Success
 from django_q.monitor import Stat
 
 logger = logging.getLogger(__name__)
@@ -21,10 +23,25 @@ def is_q_cluster_running(**kwargs):
 
     clusters = Stat.get_all()
 
-    for cluster in clusters:
-        print("Cluster:", cluster)
+    if len(clusters) > 0:
+        return True
 
-    return len(clusters) > 0
+    """
+    Sometimes Stat.get_all() returns [].
+    In this case we have the 'heartbeat' task running every five minutes.
+    Check to see if we have a result within the last ten minutes
+    """
+
+    now = datetime.now()
+    past = now - timedelta(minutes=10)
+
+    results = Success.objects.filter(
+        func='InvenTree.tasks.heartbeat',
+        started__gte=past
+    )
+
+    # If any results are returned, then the background worker is running!
+    return results.exists()
 
 
 def check_system_health(**kwargs):
diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py
index 61b6885b03..985b9e24da 100644
--- a/InvenTree/InvenTree/tasks.py
+++ b/InvenTree/InvenTree/tasks.py
@@ -6,8 +6,6 @@ import json
 import requests
 import logging
 
-from datetime import timedelta
-
 from django.core.exceptions import AppRegistryNotReady
 
 
@@ -37,6 +35,17 @@ def schedule_task(taskname, **kwargs):
         )
 
 
+def heartbeat():
+    """
+    Simple task which runs at 5 minute intervals,
+    so we can determine that the background worker
+    is actually running.
+
+    (There is probably a less "hacky" way of achieving this)
+    """
+    pass
+
+
 def delete_successful_tasks():
     """
     Delete successful task logs
@@ -45,6 +54,7 @@ def delete_successful_tasks():
 
     pass
 
+
 def check_for_updates():
     """
     Check if there is an update for InvenTree
@@ -52,7 +62,6 @@ def check_for_updates():
 
     try:
         import common.models
-        import InvenTree.version
     except AppRegistryNotReady:
         return
 
@@ -67,7 +76,7 @@ def check_for_updates():
     tag = data.get('tag_name', None)
 
     if not tag:
-        logger.warning(f"'tag_name' missing from GitHub response")
+        logger.warning("'tag_name' missing from GitHub response")
         return
 
     match = re.match(r"^.*(\d+)\.(\d+)\.(\d+).*$", tag)
diff --git a/InvenTree/templates/navbar.html b/InvenTree/templates/navbar.html
index c52e87d753..acd71f0cd8 100644
--- a/InvenTree/templates/navbar.html
+++ b/InvenTree/templates/navbar.html
@@ -59,8 +59,10 @@
           {% endif %}
           <li class='dropdown'>
             <a class='dropdown-toggle' data-toggle='dropdown' href="#">
-              {% if not system_healthy or not up_to_date %}
+              {% if not system_healthy %}
               <span class='fas fa-exclamation-triangle icon-red'></span>
+              {% elif not up_to_date %}
+              <span class='fas fa-info-circle icon-green'></span>
               {% endif %}
               <span class="fas fa-user"></span> <b>{{ user.get_username }}</b></a>
             <ul class='dropdown-menu'>