Merge branch 'dev' of gitlab.com:crafty-controller/crafty-commander into dev

app/classes/minecraft/stats.py

@@ -5,10 +5,10 @@ import base64
 import psutil
 
 from app.classes.models.management import Host_Stats
-from app.classes.models.servers import Server_Stats, servers_helper
 
 from app.classes.shared.helpers import helper
-from app.classes.minecraft.mc_ping import ping, ping_bedrock
+from app.classes.models.servers import servers_helper
+from app.classes.minecraft.mc_ping import ping
 
 logger = logging.getLogger(__name__)
 
@@ -36,8 +36,8 @@ class Stats:
             'mem_total': helper.human_readable_file_size(psutil.virtual_memory()[0]),
             'disk_data': self._all_disk_usage()
         }
-        server_stats = self.get_servers_stats()
+        #server_stats = self.get_servers_stats()
-        data['servers'] = server_stats
+        #data['servers'] = server_stats
         data['node_stats'] = node_stats
 
         return data
@@ -120,6 +120,34 @@ class Stats:
 
         return level_total_size
 
+    def get_server_players(self, server_id):
+
+        server = servers_helper.get_server_data_by_id(server_id)
+
+        logger.info(f"Getting players for server {server}")
+
+        # get our settings and data dictionaries
+        # server_settings = server.get('server_settings', {})
+        # server_data = server.get('server_data_obj', {})
+
+
+        # TODO: search server properties file for possible override of 127.0.0.1
+        internal_ip = server['server_ip']
+        server_port = server['server_port']
+
+        logger.debug("Pinging {internal_ip} on port {server_port}")
+        if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
+            int_mc_ping = ping(internal_ip, int(server_port))
+
+
+            ping_data = {}
+
+            # if we got a good ping return, let's parse it
+            if int_mc_ping:
+                ping_data = Stats.parse_server_ping(int_mc_ping)
+                return ping_data['players']
+        return []
+
     @staticmethod
     def parse_server_ping(ping_obj: object):
         online_stats = {}
@@ -168,277 +196,6 @@ class Stats:
 
         return ping_data
 
-    def get_server_players(self, server_id):
-
-        server = servers_helper.get_server_data_by_id(server_id)
-
-        logger.info(f"Getting players for server {server}")
-
-        # get our settings and data dictionaries
-        # server_settings = server.get('server_settings', {})
-        # server_data = server.get('server_data_obj', {})
-
-
-        # TODO: search server properties file for possible override of 127.0.0.1
-        internal_ip = server['server_ip']
-        server_port = server['server_port']
-
-        logger.debug("Pinging {internal_ip} on port {server_port}")
-        if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
-            int_mc_ping = ping(internal_ip, int(server_port))
-
-
-            ping_data = {}
-
-            # if we got a good ping return, let's parse it
-            if int_mc_ping:
-                ping_data = self.parse_server_ping(int_mc_ping)
-                return ping_data['players']
-        return []
-
-    def get_servers_stats(self):
-
-        server_stats_list = []
-        server_stats = {}
-
-        servers = self.controller.servers_list
-
-        logger.info("Getting Stats for all servers...")
-
-        for s in servers:
-
-            server_id = s.get('server_id', None)
-            server = servers_helper.get_server_data_by_id(server_id)
-
-
-            logger.debug(f'Getting stats for server: {server_id}')
-
-            # get our server object, settings and data dictionaries
-            server_obj = s.get('server_obj', None)
-            server_obj.reload_server_settings()
-
-            # world data
-            server_name = server['server_name']
-            server_path = server['path']
-
-            # process stats
-            p_stats = self._get_process_stats(server_obj.process)
-
-            # TODO: search server properties file for possible override of 127.0.0.1
-            internal_ip = server['server_ip']
-            server_port = server['server_port']
-            server = s.get('server_name', f"ID#{server_id}")
-
-            logger.debug("Pinging server '{server}' on {internal_ip}:{server_port}")
-            if servers_helper.get_server_type_by_id(server_id) == 'minecraft-bedrock':
-                int_mc_ping = ping_bedrock(internal_ip, int(server_port))
-            else:
-                int_mc_ping = ping(internal_ip, int(server_port))
-
-            int_data = False
-            ping_data = {}
-
-            # if we got a good ping return, let's parse it
-            if int_mc_ping:
-                int_data = True
-                if servers_helper.get_server_type_by_id(s['server_id']) == 'minecraft-bedrock':
-                    ping_data = self.parse_server_RakNet_ping(int_mc_ping)
-                else:
-                    ping_data = self.parse_server_ping(int_mc_ping)
-        #Makes sure we only show stats when a server is online otherwise people have gotten confused.
-            if server_obj.check_running():
-                server_stats = {
-                    'id': server_id,
-                    'started': server_obj.get_start_time(),
-                    'running': server_obj.check_running(),
-                    'cpu': p_stats.get('cpu_usage', 0),
-                    'mem': p_stats.get('memory_usage', 0),
-                    "mem_percent": p_stats.get('mem_percentage', 0),
-                    'world_name': server_name,
-                    'world_size': self.get_world_size(server_path),
-                    'server_port': server_port,
-                    'int_ping_results': int_data,
-                    'online': ping_data.get("online", False),
-                    "max": ping_data.get("max", False),
-                    'players': ping_data.get("players", False),
-                    'desc': ping_data.get("server_description", False),
-                    'version': ping_data.get("server_version", False)
-                }
-            else:
-                server_stats = {
-                    'id': server_id,
-                    'started': server_obj.get_start_time(),
-                    'running': server_obj.check_running(),
-                    'cpu': p_stats.get('cpu_usage', 0),
-                    'mem': p_stats.get('memory_usage', 0),
-                    "mem_percent": p_stats.get('mem_percentage', 0),
-                    'world_name': server_name,
-                    'world_size': self.get_world_size(server_path),
-                    'server_port': server_port,
-                    'int_ping_results': int_data,
-                    'online': False,
-                    "max": False,
-                    'players': False,
-                    'desc': False,
-                    'version': False
-                }
-
-            # add this servers data to the stack
-            server_stats_list.append(server_stats)
-
-        return server_stats_list
-
-    def get_raw_server_stats(self, server_id):
-
-        try:
-            self.controller.get_server_obj(server_id)
-        except:
-            return {    'id': server_id,
-                        'started': False,
-                        'running': False,
-                        'cpu': 0,
-                        'mem': 0,
-                        "mem_percent": 0,
-                        'world_name': None,
-                        'world_size': None,
-                        'server_port': None,
-                        'int_ping_results': False,
-                        'online': False,
-                        'max': False,
-                        'players': False,
-                        'desc': False,
-                        'version': False,
-                        'icon': False}
-
-        server_stats = {}
-        server = self.controller.get_server_obj(server_id)
-        if not server:
-            return {}
-        server_dt = servers_helper.get_server_data_by_id(server_id)
-
-
-        logger.debug(f'Getting stats for server: {server_id}')
-
-        # get our server object, settings and data dictionaries
-        server_obj = self.controller.get_server_obj(server_id)
-        server_obj.reload_server_settings()
-
-        # world data
-        server_name = server_dt['server_name']
-        server_path = server_dt['path']
-
-        # process stats
-        p_stats = self._get_process_stats(server_obj.process)
-
-        # TODO: search server properties file for possible override of 127.0.0.1
-        #internal_ip =   server['server_ip']
-        #server_port = server['server_port']
-        internal_ip = server_dt['server_ip']
-        server_port = server_dt['server_port']
-
-
-        logger.debug(f"Pinging server '{server.name}' on {internal_ip}:{server_port}")
-        if servers_helper.get_server_type_by_id(server_id) == 'minecraft-bedrock':
-            int_mc_ping = ping_bedrock(internal_ip, int(server_port))
-        else:
-            int_mc_ping = ping(internal_ip, int(server_port))
-
-        int_data = False
-        ping_data = {}
-        #Makes sure we only show stats when a server is online otherwise people have gotten confused.
-        if server_obj.check_running():
-            # if we got a good ping return, let's parse it
-            if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
-                if int_mc_ping:
-                    int_data = True
-                    ping_data = self.parse_server_ping(int_mc_ping)
-
-                server_stats = {
-                    'id': server_id,
-                    'started': server_obj.get_start_time(),
-                    'running': server_obj.check_running(),
-                    'cpu': p_stats.get('cpu_usage', 0),
-                    'mem': p_stats.get('memory_usage', 0),
-                    "mem_percent": p_stats.get('mem_percentage', 0),
-                    'world_name': server_name,
-                    'world_size': self.get_world_size(server_path),
-                    'server_port': server_port,
-                    'int_ping_results': int_data,
-                    'online': ping_data.get("online", False),
-                    "max": ping_data.get("max", False),
-                    'players': ping_data.get("players", False),
-                    'desc': ping_data.get("server_description", False),
-                    'version': ping_data.get("server_version", False),
-                    'icon': ping_data.get("server_icon", False)
-                }
-
-            else:
-                if int_mc_ping:
-                    int_data = True
-                    ping_data = self.parse_server_RakNet_ping(int_mc_ping)
-                    try:
-                        server_icon = base64.encodebytes(ping_data['icon'])
-                    except  Exception as e:
-                        server_icon = False
-                        logger.info(f"Unable to read the server icon : {e}")
-
-                    server_stats = {
-                        'id': server_id,
-                        'started': server_obj.get_start_time(),
-                        'running': server_obj.check_running(),
-                        'cpu': p_stats.get('cpu_usage', 0),
-                        'mem': p_stats.get('memory_usage', 0),
-                        "mem_percent": p_stats.get('mem_percentage', 0),
-                        'world_name': server_name,
-                        'world_size': self.get_world_size(server_path),
-                        'server_port': server_port,
-                        'int_ping_results': int_data,
-                        'online': ping_data['online'],
-                        'max': ping_data['max'],
-                        'players': [],
-                        'desc': ping_data['server_description'],
-                        'version': ping_data['server_version'],
-                        'icon': server_icon
-                    }
-                else:
-                    server_stats = {
-                        'id': server_id,
-                        'started': server_obj.get_start_time(),
-                        'running': server_obj.check_running(),
-                        'cpu': p_stats.get('cpu_usage', 0),
-                        'mem': p_stats.get('memory_usage', 0),
-                        "mem_percent": p_stats.get('mem_percentage', 0),
-                        'world_name': server_name,
-                        'world_size': self.get_world_size(server_path),
-                        'server_port': server_port,
-                        'int_ping_results': int_data,
-                        'online': False,
-                        'max': False,
-                        'players': False,
-                        'desc': False,
-                        'version': False,
-                        'icon': False
-                    }
-        else:
-            server_stats = {
-                'id': server_id,
-                'started': server_obj.get_start_time(),
-                'running': server_obj.check_running(),
-                'cpu': p_stats.get('cpu_usage', 0),
-                'mem': p_stats.get('memory_usage', 0),
-                "mem_percent": p_stats.get('mem_percentage', 0),
-                'world_name': server_name,
-                'world_size': self.get_world_size(server_path),
-                'server_port': server_port,
-                'int_ping_results': int_data,
-                'online': False,
-                "max": False,
-                'players': False,
-                'desc': False,
-                'version': False
-            }
-
-        return server_stats
 
     def record_stats(self):
         stats_to_send = self.get_node_stats()
@@ -456,26 +213,26 @@ class Stats:
             Host_Stats.disk_json: node_stats.get('disk_data', '{}')
         }).execute()
 
-        server_stats = stats_to_send.get('servers')
+#        server_stats = stats_to_send.get('servers')#
-
+#
-        for server in server_stats:
+#        for server in server_stats:
-            Server_Stats.insert({
+#            Server_Stats.insert({
-                Server_Stats.server_id: server.get('id', 0),
+#                Server_Stats.server_id: server.get('id', 0),
-                Server_Stats.started: server.get('started', ""),
+#                Server_Stats.started: server.get('started', ""),
-                Server_Stats.running: server.get('running', False),
+#                Server_Stats.running: server.get('running', False),
-                Server_Stats.cpu: server.get('cpu', 0),
+#                Server_Stats.cpu: server.get('cpu', 0),
-                Server_Stats.mem: server.get('mem', 0),
+#                Server_Stats.mem: server.get('mem', 0),
-                Server_Stats.mem_percent: server.get('mem_percent', 0),
+#                Server_Stats.mem_percent: server.get('mem_percent', 0),
-                Server_Stats.world_name: server.get('world_name', ""),
+#                Server_Stats.world_name: server.get('world_name', ""),
-                Server_Stats.world_size: server.get('world_size', ""),
+#                Server_Stats.world_size: server.get('world_size', ""),
-                Server_Stats.server_port: server.get('server_port', ""),
+#                Server_Stats.server_port: server.get('server_port', ""),
-                Server_Stats.int_ping_results: server.get('int_ping_results', False),
+#                Server_Stats.int_ping_results: server.get('int_ping_results', False),
-                Server_Stats.online: server.get("online", False),
+#                Server_Stats.online: server.get("online", False),
-                Server_Stats.max: server.get("max", False),
+#                Server_Stats.max: server.get("max", False),
-                Server_Stats.players: server.get("players", False),
+#                Server_Stats.players: server.get("players", False),
-                Server_Stats.desc: server.get("desc", False),
+#                Server_Stats.desc: server.get("desc", False),
-                Server_Stats.version: server.get("version", False)
+#                Server_Stats.version: server.get("version", False)
-            }).execute()
+#            }).execute()
 
         # delete old data
         max_age = helper.get_setting("history_max_age")
@@ -483,4 +240,4 @@ class Stats:
         last_week = now.day - max_age
 
         Host_Stats.delete().where(Host_Stats.time < last_week).execute()
-        Server_Stats.delete().where(Server_Stats.created < last_week).execute()
+#        Server_Stats.delete().where(Server_Stats.created < last_week).execute()

app/classes/shared/server.py

@@ -3,6 +3,7 @@ import sys
 import re
 import time
 import datetime
+import base64
 import threading
 import logging.config
 import subprocess
@@ -12,11 +13,14 @@ from apscheduler.schedulers.background import BackgroundScheduler
 #TZLocal is set as a hidden import on win pipeline
 from tzlocal import get_localzone
 
-from app.classes.models.servers import servers_helper
+from app.classes.models.servers import Server_Stats, servers_helper
 from app.classes.models.management import management_helper
 from app.classes.models.users import users_helper
 from app.classes.models.server_permissions import server_permissions
 
+from app.classes.minecraft.stats import Stats
+from app.classes.minecraft.mc_ping import ping, ping_bedrock
+
 from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
 from app.classes.shared.translation import translation
@@ -96,6 +100,9 @@ class ServerOutBuf:
         )
 
 
+#************************************************************************************************
+#                               Minecraft Server Class
+#************************************************************************************************
 class Server:
 
     def __init__(self, stats):
@@ -123,6 +130,10 @@ class Server:
         servers_helper.server_crash_reset(self.server_id)
         servers_helper.set_update(self.server_id, False)
 
+
+#************************************************************************************************
+#                               Minecraft Server Management
+#************************************************************************************************
     def reload_server_settings(self):
         server_data = servers_helper.get_server_data_by_id(self.server_id)
         self.settings = server_data
@@ -137,6 +148,8 @@ class Server:
         self.name = serverName
         self.settings = server_data_obj
 
+        self.record_server_stats()
+
         # build our server run command
 
         if server_data_obj['auto_start']:
@@ -162,6 +175,12 @@ class Server:
         self.server_thread = threading.Thread(target=self.start_server, daemon=True, args=(user_id,), name=f'{self.server_id}_server_thread')
         self.server_thread.start()
 
+        #Register an shedule for polling server stats when running
+        logger.info(f"Polling server statistics {self.name} every {5} seconds")
+        console.info(f"Polling server statistics {self.name} every {5} seconds")
+        self.server_scheduler.add_job(self.realtime_stats, 'interval', seconds=5, id="stats_"+str(self.server_id))
+
+
     def setup_server_run_command(self):
         # configure the server
         server_exec_path = helper.get_os_understandable_path(self.settings['executable'])
@@ -310,7 +329,7 @@ class Server:
             console.info(f"Server {self.name} running with PID {self.process.pid}")
             self.is_crashed = False
             servers_helper.server_crash_reset(self.server_id)
-            self.stats.record_stats()
+            self.record_server_stats()
             check_internet_thread = threading.Thread(
                 target=self.check_internet_thread, daemon=True, args=(user_id, user_lang, ), name=f"{self.name}_Internet")
             check_internet_thread.start()
@@ -417,7 +436,10 @@ class Server:
         self.cleanup_server_object()
         server_users = server_permissions.get_server_user_list(self.server_id)
 
-        self.stats.record_stats()
+        # remove the stats polling job since server is stopped
+        self.server_scheduler.remove_job("stats_"+str(self.server_id))
+
+        self.record_server_stats()
 
         for user in server_users:
             websocket_helper.broadcast_user(user, 'send_start_reload', {
@@ -748,3 +770,363 @@ class Server:
                     websocket_helper.broadcast_user(user,'notification',
                                            "Executable update failed for " + self.name + ". Check log file for details.")
                 logger.error("Executable download failed.")
+
+
+
+#************************************************************************************************
+#                               Minecraft Servers Statistics
+#************************************************************************************************
+
+    def realtime_stats(self):
+        total_players = 0
+        max_players = 0
+        servers_ping = []
+        raw_ping_result = []
+        raw_ping_result = self.get_raw_server_stats(self.server_id)
+
+        if f"{raw_ping_result.get('icon')}" == "b''":
+            raw_ping_result['icon'] = False
+
+        servers_ping.append({
+            'id': raw_ping_result.get('id'),
+            'started': raw_ping_result.get('started'),
+            'running': raw_ping_result.get('running'),
+            'cpu': raw_ping_result.get('cpu'),
+            'mem': raw_ping_result.get('mem'),
+            'mem_percent': raw_ping_result.get('mem_percent'),
+            'world_name': raw_ping_result.get('world_name'),
+            'world_size': raw_ping_result.get('world_size'),
+            'server_port': raw_ping_result.get('server_port'),
+            'int_ping_results': raw_ping_result.get('int_ping_results'),
+            'online': raw_ping_result.get('online'),
+            'max': raw_ping_result.get('max'),
+            'players': raw_ping_result.get('players'),
+            'desc': raw_ping_result.get('desc'),
+            'version': raw_ping_result.get('version'),
+            'icon': raw_ping_result.get('icon')
+        })
+        if len(websocket_helper.clients) > 0:
+            websocket_helper.broadcast_page_params(
+                '/panel/server_detail',
+                {
+                    'id': str(self.server_id)
+                },
+                'update_server_details',
+                {
+                    'id': raw_ping_result.get('id'),
+                    'started': raw_ping_result.get('started'),
+                    'running': raw_ping_result.get('running'),
+                    'cpu': raw_ping_result.get('cpu'),
+                    'mem': raw_ping_result.get('mem'),
+                    'mem_percent': raw_ping_result.get('mem_percent'),
+                    'world_name': raw_ping_result.get('world_name'),
+                    'world_size': raw_ping_result.get('world_size'),
+                    'server_port': raw_ping_result.get('server_port'),
+                    'int_ping_results': raw_ping_result.get('int_ping_results'),
+                    'online': raw_ping_result.get('online'),
+                    'max': raw_ping_result.get('max'),
+                    'players': raw_ping_result.get('players'),
+                    'desc': raw_ping_result.get('desc'),
+                    'version': raw_ping_result.get('version'),
+                    'icon': raw_ping_result.get('icon')
+                }
+            )
+        total_players += int(raw_ping_result.get('online'))
+        max_players += int(raw_ping_result.get('max'))
+
+        self.record_server_stats()
+
+        if (len(servers_ping) > 0) & (len(websocket_helper.clients) > 0):
+            try:
+                websocket_helper.broadcast_page('/panel/dashboard', 'update_server_status', servers_ping)
+                websocket_helper.broadcast_page('/status', 'update_server_status', servers_ping)
+            except:
+                console.warning("Can't broadcast server status to websocket")
+
+    def get_servers_stats(self):
+
+        server_stats = {}
+
+        logger.info("Getting Stats for Server " + self.name + " ...")
+
+        server_id = self.server_id
+        server = servers_helper.get_server_data_by_id(server_id)
+
+        logger.debug(f'Getting stats for server: {server_id}')
+
+        # get our server object, settings and data dictionaries
+        self.reload_server_settings()
+
+        # world data
+        server_path = server['path']
+
+        # process stats
+        p_stats = Stats._get_process_stats(self.process)
+
+        # TODO: search server properties file for possible override of 127.0.0.1
+        internal_ip = server['server_ip']
+        server_port = server['server_port']
+        server_name = server.get('server_name', f"ID#{server_id}")
+
+        logger.debug("Pinging server '{server}' on {internal_ip}:{server_port}")
+        if servers_helper.get_server_type_by_id(server_id) == 'minecraft-bedrock':
+            int_mc_ping = ping_bedrock(internal_ip, int(server_port))
+        else:
+            int_mc_ping = ping(internal_ip, int(server_port))
+
+        int_data = False
+        ping_data = {}
+
+        # if we got a good ping return, let's parse it
+        if int_mc_ping:
+            int_data = True
+            if servers_helper.get_server_type_by_id(server['server_id']) == 'minecraft-bedrock':
+                ping_data = Stats.parse_server_RakNet_ping(int_mc_ping)
+            else:
+                ping_data = Stats.parse_server_ping(int_mc_ping)
+    #Makes sure we only show stats when a server is online otherwise people have gotten confused.
+        if self.check_running():
+            server_stats = {
+                'id': server_id,
+                'started': self.get_start_time(),
+                'running': self.check_running(),
+                'cpu': p_stats.get('cpu_usage', 0),
+                'mem': p_stats.get('memory_usage', 0),
+                "mem_percent": p_stats.get('mem_percentage', 0),
+                'world_name': server_name,
+                'world_size': Stats.get_world_size(server_path),
+                'server_port': server_port,
+                'int_ping_results': int_data,
+                'online': ping_data.get("online", False),
+                "max": ping_data.get("max", False),
+                'players': ping_data.get("players", False),
+                'desc': ping_data.get("server_description", False),
+                'version': ping_data.get("server_version", False)
+            }
+        else:
+            server_stats = {
+                'id': server_id,
+                'started': self.get_start_time(),
+                'running': self.check_running(),
+                'cpu': p_stats.get('cpu_usage', 0),
+                'mem': p_stats.get('memory_usage', 0),
+                "mem_percent": p_stats.get('mem_percentage', 0),
+                'world_name': server_name,
+                'world_size': Stats.get_world_size(server_path),
+                'server_port': server_port,
+                'int_ping_results': int_data,
+                'online': False,
+                "max": False,
+                'players': False,
+                'desc': False,
+                'version': False
+            }
+
+        return server_stats
+
+    def get_server_players(self):
+
+        server = servers_helper.get_server_data_by_id(self.server_id)
+
+        logger.info(f"Getting players for server {server}")
+
+        # get our settings and data dictionaries
+        # server_settings = server.get('server_settings', {})
+        # server_data = server.get('server_data_obj', {})
+
+
+        # TODO: search server properties file for possible override of 127.0.0.1
+        internal_ip = server['server_ip']
+        server_port = server['server_port']
+
+        logger.debug("Pinging {internal_ip} on port {server_port}")
+        if servers_helper.get_server_type_by_id(self.server_id) != 'minecraft-bedrock':
+            int_mc_ping = ping(internal_ip, int(server_port))
+
+
+            ping_data = {}
+
+            # if we got a good ping return, let's parse it
+            if int_mc_ping:
+                ping_data = Stats.parse_server_ping(int_mc_ping)
+                return ping_data['players']
+        return []
+
+    def get_raw_server_stats(self, server_id):
+
+        try:
+            server = servers_helper.get_server_obj(server_id)
+        except:
+            return {'id': server_id,
+                    'started': False,
+                    'running': False,
+                    'cpu': 0,
+                    'mem': 0,
+                    "mem_percent": 0,
+                    'world_name': None,
+                    'world_size': None,
+                    'server_port': None,
+                    'int_ping_results': False,
+                    'online': False,
+                    'max': False,
+                    'players': False,
+                    'desc': False,
+                    'version': False,
+                    'icon': False}
+
+        server_stats = {}
+        server = servers_helper.get_server_obj(server_id)
+        if not server:
+            return {}
+        server_dt = servers_helper.get_server_data_by_id(server_id)
+
+
+        logger.debug(f'Getting stats for server: {server_id}')
+
+        # get our server object, settings and data dictionaries
+        self.reload_server_settings()
+
+        # world data
+        server_name = server_dt['server_name']
+        server_path = server_dt['path']
+
+        # process stats
+        p_stats = Stats._get_process_stats(self.process)
+
+        # TODO: search server properties file for possible override of 127.0.0.1
+        #internal_ip =   server['server_ip']
+        #server_port = server['server_port']
+        internal_ip = server_dt['server_ip']
+        server_port = server_dt['server_port']
+
+
+        logger.debug(f"Pinging server '{self.name}' on {internal_ip}:{server_port}")
+        if servers_helper.get_server_type_by_id(server_id) == 'minecraft-bedrock':
+            int_mc_ping = ping_bedrock(internal_ip, int(server_port))
+        else:
+            int_mc_ping = ping(internal_ip, int(server_port))
+
+        int_data = False
+        ping_data = {}
+        #Makes sure we only show stats when a server is online otherwise people have gotten confused.
+        if self.check_running():
+            # if we got a good ping return, let's parse it
+            if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
+                if int_mc_ping:
+                    int_data = True
+                    ping_data = Stats.parse_server_ping(int_mc_ping)
+
+                server_stats = {
+                    'id': server_id,
+                    'started': self.get_start_time(),
+                    'running': self.check_running(),
+                    'cpu': p_stats.get('cpu_usage', 0),
+                    'mem': p_stats.get('memory_usage', 0),
+                    "mem_percent": p_stats.get('mem_percentage', 0),
+                    'world_name': server_name,
+                    'world_size': Stats.get_world_size(server_path),
+                    'server_port': server_port,
+                    'int_ping_results': int_data,
+                    'online': ping_data.get("online", False),
+                    "max": ping_data.get("max", False),
+                    'players': ping_data.get("players", False),
+                    'desc': ping_data.get("server_description", False),
+                    'version': ping_data.get("server_version", False),
+                    'icon': ping_data.get("server_icon", False)
+                }
+
+            else:
+                if int_mc_ping:
+                    int_data = True
+                    ping_data = Stats.parse_server_RakNet_ping(int_mc_ping)
+                    try:
+                        server_icon = base64.encodebytes(ping_data['icon'])
+                    except  Exception as ex:
+                        server_icon = False
+                        logger.info(f"Unable to read the server icon : {ex}")
+
+                    server_stats = {
+                        'id': server_id,
+                        'started': self.get_start_time(),
+                        'running': self.check_running(),
+                        'cpu': p_stats.get('cpu_usage', 0),
+                        'mem': p_stats.get('memory_usage', 0),
+                        "mem_percent": p_stats.get('mem_percentage', 0),
+                        'world_name': server_name,
+                        'world_size': Stats.get_world_size(server_path),
+                        'server_port': server_port,
+                        'int_ping_results': int_data,
+                        'online': ping_data['online'],
+                        'max': ping_data['max'],
+                        'players': [],
+                        'desc': ping_data['server_description'],
+                        'version': ping_data['server_version'],
+                        'icon': server_icon
+                    }
+                else:
+                    server_stats = {
+                        'id': server_id,
+                        'started': self.get_start_time(),
+                        'running': self.check_running(),
+                        'cpu': p_stats.get('cpu_usage', 0),
+                        'mem': p_stats.get('memory_usage', 0),
+                        "mem_percent": p_stats.get('mem_percentage', 0),
+                        'world_name': server_name,
+                        'world_size': Stats.get_world_size(server_path),
+                        'server_port': server_port,
+                        'int_ping_results': int_data,
+                        'online': False,
+                        'max': False,
+                        'players': False,
+                        'desc': False,
+                        'version': False,
+                        'icon': False
+                    }
+        else:
+            server_stats = {
+                'id': server_id,
+                'started': self.get_start_time(),
+                'running': self.check_running(),
+                'cpu': p_stats.get('cpu_usage', 0),
+                'mem': p_stats.get('memory_usage', 0),
+                "mem_percent": p_stats.get('mem_percentage', 0),
+                'world_name': server_name,
+                'world_size': Stats.get_world_size(server_path),
+                'server_port': server_port,
+                'int_ping_results': int_data,
+                'online': False,
+                "max": False,
+                'players': False,
+                'desc': False,
+                'version': False
+            }
+
+        return server_stats
+
+    def record_server_stats(self):
+
+        server = self.get_servers_stats()
+        Server_Stats.insert({
+            Server_Stats.server_id: server.get('id', 0),
+            Server_Stats.started: server.get('started', ""),
+            Server_Stats.running: server.get('running', False),
+            Server_Stats.cpu: server.get('cpu', 0),
+            Server_Stats.mem: server.get('mem', 0),
+            Server_Stats.mem_percent: server.get('mem_percent', 0),
+            Server_Stats.world_name: server.get('world_name', ""),
+            Server_Stats.world_size: server.get('world_size', ""),
+            Server_Stats.server_port: server.get('server_port', ""),
+            Server_Stats.int_ping_results: server.get('int_ping_results', False),
+            Server_Stats.online: server.get("online", False),
+            Server_Stats.max: server.get("max", False),
+            Server_Stats.players: server.get("players", False),
+            Server_Stats.desc: server.get("desc", False),
+            Server_Stats.version: server.get("version", False)
+        }).execute()
+
+        # delete old data
+        max_age = helper.get_setting("history_max_age")
+        now = datetime.datetime.now()
+        last_week = now.day - max_age
+
+        Server_Stats.delete().where(Server_Stats.created < last_week).execute()

app/classes/shared/tasks.py

@@ -16,8 +16,6 @@ from app.classes.web.websocket_helper import websocket_helper
 from app.classes.minecraft.serverjars import server_jar_obj
 from app.classes.models.management import management_helper
 from app.classes.controllers.users_controller import Users_Controller
-from app.classes.controllers.servers_controller import Servers_Controller
-from app.classes.models.servers import servers_helper
 from app.classes.models.users import users_helper
 
 logger = logging.getLogger('apscheduler')
@@ -454,79 +452,6 @@ class TasksManager:
                         'mem_percent': host_stats.get('mem_percent'),
                         'mem_usage': host_stats.get('mem_usage')
                     })
-
-            for user in Users_Controller.get_all_users():
-                total_players = 0
-                max_players = 0
-                servers_ping = []
-                players_ping = {}
-                if user.superuser:
-                    servers = Servers_Controller.get_all_servers_stats()
-                else:
-                    servers = Servers_Controller.get_authorized_servers_stats(user.user_id)
-                for srv in servers:
-                    if srv:
-                        server_id = srv['server_data']['server_id']
-                        srv['raw_ping_result'] = self.controller.stats.get_raw_server_stats(server_id)
-
-                        servers_ping.append({
-                            'id': srv['raw_ping_result'].get('id'),
-                            'started': srv['raw_ping_result'].get('started'),
-                            'running': srv['raw_ping_result'].get('running'),
-                            'cpu': srv['raw_ping_result'].get('cpu'),
-                            'mem': srv['raw_ping_result'].get('mem'),
-                            'mem_percent': srv['raw_ping_result'].get('mem_percent'),
-                            'world_name': srv['raw_ping_result'].get('world_name'),
-                            'world_size': srv['raw_ping_result'].get('world_size'),
-                            'server_port': srv['raw_ping_result'].get('server_port'),
-                            'int_ping_results': srv['raw_ping_result'].get('int_ping_results'),
-                            'online': srv['raw_ping_result'].get('online'),
-                            'max': srv['raw_ping_result'].get('max'),
-                            'players': srv['raw_ping_result'].get('players'),
-                            'desc': srv['raw_ping_result'].get('desc'),
-                            'version': srv['raw_ping_result'].get('version'),
-                            'crashed': servers_helper.is_crashed(server_id),
-                        })
-                        if len(websocket_helper.clients) > 0:
-                            websocket_helper.broadcast_user_page_params(
-                                '/panel/server_detail',
-                                {
-                                    'id': str(server_id)
-                                }, user.user_id,
-                                'update_server_details',
-                                {
-                                    'id': srv['raw_ping_result'].get('id'),
-                                    'started': srv['raw_ping_result'].get('started'),
-                                    'running': srv['raw_ping_result'].get('running'),
-                                    'cpu': srv['raw_ping_result'].get('cpu'),
-                                    'mem': srv['raw_ping_result'].get('mem'),
-                                    'mem_percent': srv['raw_ping_result'].get('mem_percent'),
-                                    'world_name': srv['raw_ping_result'].get('world_name'),
-                                    'world_size': srv['raw_ping_result'].get('world_size'),
-                                    'server_port': srv['raw_ping_result'].get('server_port'),
-                                    'int_ping_results': srv['raw_ping_result'].get('int_ping_results'),
-                                    'online': srv['raw_ping_result'].get('online'),
-                                    'max': srv['raw_ping_result'].get('max'),
-                                    'players': srv['raw_ping_result'].get('players'),
-                                    'desc': srv['raw_ping_result'].get('desc'),
-                                    'version': srv['raw_ping_result'].get('version'),
-                                    'crashed': servers_helper.is_crashed(server_id),
-                                }
-                            )
-                        total_players += int(srv['raw_ping_result'].get('online'))
-                        max_players += int(srv['raw_ping_result'].get('max'))
-                players_ping = {
-                        'total_players': total_players,
-                        'max_players': max_players
-                    }
-                websocket_helper.broadcast_user_page('/panel/dashboard', user.user_id, 'update_player_status', players_ping)
-
-                if (len(servers_ping) > 0) & (len(websocket_helper.clients) > 0):
-                    try:
-                        websocket_helper.broadcast_user_page('/panel/dashboard', user.user_id, 'update_server_status', servers_ping)
-                        websocket_helper.broadcast_page('/status', 'update_server_status', servers_ping)
-                    except:
-                        console.warning("Can't broadcast server status to websocket")
             time.sleep(2)
 
     def log_watcher(self):

app/classes/web/panel_handler.py

@@ -12,7 +12,6 @@ import requests
 import tornado.web
 import tornado.escape
 from tornado import iostream
-from tornado.ioloop import IOLoop
 
 #TZLocal is set as a hidden import on win pipeline
 from tzlocal import get_localzone
@@ -374,12 +373,20 @@ class PanelHandler(BaseHandler):
                     user_order.remove(server_id)
             page_data['servers'] = page_servers
 
-            try:
-                self.fetch_server_data(page_data)
-            except:
-                page_data['num_players'] = 0
 
-            IOLoop.current().add_callback(self.fetch_server_data, page_data)
+            for data in page_data['servers']:
+                data['stats']['crashed'] = self.controller.servers.is_crashed(
+                        str(data['stats']['server_id']['server_id']))
+                try:
+                    data['stats']['waiting_start'] = self.controller.servers.get_waiting_start(
+                        str(data['stats']['server_id']['server_id']))
+                except Exception as e:
+                    logger.error(f"Failed to get server waiting to start: {e}")
+                    data['stats']['waiting_start'] = False
+
+            #num players is set to zero here. If we poll all servers while dashboard is loading it takes FOREVER. We leave this to the
+            #async polling once dashboard is served.
+            page_data['num_players'] = 0
 
             template = "panel/dashboard.html"
 

app/classes/web/status_handler.py

@@ -13,7 +13,7 @@ class StatusHandler(BaseHandler):
         for srv in page_data['servers']:
             server_data = srv.get('server_data', False)
             server_id = server_data.get('server_id', False)
-            srv['raw_ping_result'] = self.controller.stats.get_raw_server_stats(server_id)
+            srv['raw_ping_result'] = self.controller.servers.get_server_stats_by_id(server_id)
 
         template = 'public/status.html'
 
@@ -28,7 +28,7 @@ class StatusHandler(BaseHandler):
         for srv in page_data['servers']:
             server_data = srv.get('server_data', False)
             server_id = server_data.get('server_id', False)
-            srv['raw_ping_result'] = self.controller.stats.get_raw_server_stats(server_id)
+            srv['raw_ping_result'] = self.controller.servers.get_server_stats_by_id(server_id)
 
         template = 'public/status.html'
 

app/frontend/templates/panel/dashboard.html

@@ -13,7 +13,10 @@
   <div class="row page-title-header">
     <div class="col-12">
       <div class="page-header">
-        <h4 class="page-title">{{ translate('dashboard', 'dashboard', data['lang']) }}</h4>
+        <h4 class="page-title">{{ translate('dashboard', 'dashboard', data['lang']) }} 
+          {% if data['server_stats']['running'] != 0 %}
+          <span id="sync" style="margin-left: 5px;"><i class="fas fa-sync fa-spin"></i></span></h4>
+          {% end %}
       </div>
     </div>
 
@@ -69,7 +72,6 @@
                   <h5 class="mb-1 font-weight-medium text-primary">{{ translate('dashboard', 'players', data['lang']) }}
                   </h5>
                   <h3 class="mb-0 font-weight-semibold" id="total_players">{{ data['num_players'] }}</h3>
-
                 </div>
                 <div class="wrapper my-auto ml-auto ml-lg-4">
                   <p class="mb-0 text-warning"><span id="max_players">0</span> {{ translate('dashboard', 'max', data['lang']) }}</p>
@@ -155,8 +157,8 @@
                       data['lang']) }}</i></a>
                     {% elif server['stats']['waiting_start']%}
                     <!-- WHAT HAPPENED HERE -->
-                    <a data-id="{{server['server_data']['server_id']}}" class="" title={{
+                    <a data-id="{{server['server_data']['server_id']}}" class="" title="{{
-                      translate('dashboard', 'delay-explained' , data['lang'])}}>{{ translate('dashboard', 'starting',
+                      translate('dashboard', 'delay-explained' , data['lang'])}}">{{ translate('dashboard', 'starting',
                       data['lang']) }}</i></a>
                       {% elif server['stats']['downloading']%}
                       <a data-id="{{server['server_data']['server_id']}}" class=""><i class="fa fa-spinner fa-spin"></i> {{ translate('serverTerm', 'downloading',
@@ -247,6 +249,7 @@
                       data['lang']) }}</span>
                     {% end %}
                   </td>
+                  <span class="server-player-totals" id="server_players_{{server['server_data']['server_id']}}" data-players="{{ server['stats']['online']}}" data-max="{{ server['stats']['max'] }}"></span>
                 </tr>
                 {% end %}
 
@@ -351,6 +354,8 @@
     server_world = document.getElementById('server_world_' + server.id);
     server_desc = document.getElementById('server_desc_' + server.id);
     server_online_status = document.getElementById('server_running_status_' + server.id);
+    server_players = document.getElementById('server_players_' + server.id);
+    total_players = document.getElementById('total_players');
 
     console.log("Received Data : " + server.id + ": " + server);
     /* TODO Update each element */
@@ -407,6 +412,24 @@
       if (server.players) {
         server_desc.innerHTML = server.online + ` / ` + server.max + ` {{ translate('dashboard', 'max', data['lang']) }}<br />`
 
+        server_players.setAttribute('data-players', server.online);
+        server_players.setAttribute('data-max', server.max);
+        let servers = document.getElementsByClassName("server-player-totals");
+        let all_total_players = 0;
+        let all_total_max_players = 0;
+        for(var i = 0; i < servers.length; i++){
+          try{
+            all_total_players += parseInt(servers[i].getAttribute('data-players'));
+            all_total_max_players += parseInt(servers[i].getAttribute('data-max'));
+          }catch{
+            console.log("Player totals are not of type int");
+          }
+        }
+        total_players.innerHTML = all_total_players;
+        document.getElementById('max_players').innerHTML = all_total_max_players;
+        document.getElementById('sync').innerHTML = '';
+
+
         server_infos = "";
         server_infos = server.online + " / " + server.max + " {{ translate('dashboard', 'max', data['lang']) }}<br />"
       }
@@ -441,10 +464,7 @@
   }
 
   function update_servers_status(data) {
-    console.log(data);
+    update_one_server_status(data[0]);
-    for (server of data) {
-      update_one_server_status(server);
-    }
     display_motd();
   }
 
@@ -517,8 +537,7 @@
       cpu_usage = document.getElementById('cpu_usage');
       mem_usage = document.getElementById('mem_usage');
       mem_percent = document.getElementById('mem_percent');
-      total_players = document.getElementById('total_players');
+      
-      max_players = document.getElementById('max_players');
 
       webSocket.on('update_host_stats', function (hostStats) {
         var cpuDataTitle = `{% raw translate('dashboard', 'cpuCores', data['lang']) %}: ${hostStats.cpu_cores} <br /> {% raw translate("dashboard", "cpuCurFreq", data['lang']) %}: ${hostStats.cpu_cur_freq} <br /> {% raw translate("dashboard", "cpuMaxFreq", data['lang']) %}: ${hostStats.cpu_max_freq}`;
@@ -527,10 +546,6 @@
         mem_usage.setAttribute('data-original-title', `{% raw translate("dashboard", "memUsage", data['lang']) %}: ${hostStats.mem_usage}`);
         mem_percent.textContent = hostStats.mem_percent + '%';
       });
-      webSocket.on('update_player_status', function (playerStats) {
-        total_players.innerHTML = playerStats.total_players;
-        max_players.innerHTML = playerStats.max_players;
-      });
     }
 
     if (webSocket) {

app/frontend/templates/public/status.html

@@ -143,9 +143,9 @@
   }
 
   function update_servers_status(data) {
-    for (server of data) {
+    console.log(data);
-      update_one_server_status(server);
+    update_one_server_status(data[0]);
-    }
+    display_motd();
   }
 
   $(document).ready(function () {

app/frontend/templates/server/wizard.html

@@ -639,8 +639,8 @@ function hide(event) {
       cSelect.remove(0);
     }
     var newOption;
-    // create new options ordered by descending
+    // create new options ordered by ascending
-    for (var i=(cList.length)-1; i>=0; i--) {
+    for (var i=0; i < (cList.length); i++) {
       newOption = document.createElement("option");
       newOption.value = which+"|"+cList[i];  // assumes option string and value are the same
       newOption.text=cList[i];