Added crafty specific stuff to config.ini

app/classes/minecraft/controller.py

@@ -64,6 +64,7 @@ class Controller:
     @staticmethod
     def list_defined_servers():
         servers = db_helper.get_all_defined_servers()
+        return servers
 
     def list_running_servers(self):
         running_servers = []

app/classes/minecraft/mc_ping.py

@@ -0,0 +1,127 @@
+import struct
+import socket
+import base64
+import json
+import sys
+import logging.config
+
+logger = logging.getLogger(__name__)
+
+
+class Server:
+    def __init__(self, data):
+        self.description = data.get('description')
+        # print(self.description)
+        if isinstance(self.description, dict):
+
+            # cat server
+            if "translate" in self.description:
+                self.description = self.description['translate']
+
+            # waterfall / bungee
+            elif 'extra' in self.description:
+                lines = []
+
+                description = self.description
+                if 'extra' in description.keys():
+                    for e in description['extra']:
+                        if "text" in e.keys():
+                            lines.append(e['text'])
+
+                total_text = " ".join(lines)
+                self.description = total_text
+
+            # normal MC
+            else:
+                self.description = self.description['text']
+
+        self.icon = base64.b64decode(data.get('favicon', '')[22:])
+        self.players = Players(data['players']).report()
+        self.version = data['version']['name']
+        self.protocol = data['version']['protocol']
+
+
+class Players(list):
+    def __init__(self, data):
+        super().__init__(Player(x) for x in data.get('sample', []))
+        self.max = data['max']
+        self.online = data['online']
+
+    def report(self):
+        players = []
+
+        for x in self:
+            players.append(str(x))
+
+        r_data = {
+            'online': self.online,
+            'max': self.max,
+            'players': players
+        }
+
+        return json.dumps(r_data)
+
+
+class Player:
+    def __init__(self, data):
+        self.id = data['id']
+        self.name = data['name']
+
+    def __str__(self):
+        return self.name
+
+
+# For the rest of requests see wiki.vg/Protocol
+def ping(ip, port=25565):
+    def read_var_int():
+        i = 0
+        j = 0
+        while True:
+            k = sock.recv(1)
+            if not k:
+                return 0
+            k = k[0]
+            i |= (k & 0x7f) << (j * 7)
+            j += 1
+            if j > 5:
+                raise ValueError('var_int too big')
+            if not (k & 0x80):
+                return i
+
+    sock = socket.socket()
+    try:
+        sock.connect((ip, port))
+    except:
+        pass
+        return False
+
+    try:
+        host = ip.encode('utf-8')
+        data = b''  # wiki.vg/Server_List_Ping
+        data += b'\x00'  # packet ID
+        data += b'\x04'  # protocol variant
+        data += struct.pack('>b', len(host)) + host
+        data += struct.pack('>H', port)
+        data += b'\x01'  # next state
+        data = struct.pack('>b', len(data)) + data
+        sock.sendall(data + b'\x01\x00')  # handshake + status ping
+        length = read_var_int()  # full packet length
+        if length < 10:
+            if length < 0:
+                return False
+            else:
+                return False
+
+        sock.recv(1)  # packet type, 0 for pings
+        length = read_var_int()  # string length
+        data = b''
+        while len(data) != length:
+            chunk = sock.recv(length - len(data))
+            if not chunk:
+                return False
+
+            data += chunk
+        logger.debug("Server reports this data on ping: {}".format(data))
+        return Server(json.loads(data))
+    finally:
+        sock.close()

app/classes/minecraft/stats.py

@@ -0,0 +1,229 @@
+import os
+import json
+import time
+import psutil
+#import requests
+import logging
+import datetime
+
+
+from app.classes.shared.helpers import helper
+from app.classes.minecraft.mc_ping import ping
+from app.classes.minecraft.controller import controller
+from app.classes.shared.models import Host_Stats
+
+logger = logging.getLogger(__name__)
+
+
+class Stats:
+
+    def get_node_stats(self):
+        boot_time = datetime.datetime.fromtimestamp(psutil.boot_time())
+        data = {}
+        node_stats = {
+            'boot_time': str(boot_time),
+            'cpu_usage': psutil.cpu_percent(interval=0.5) / psutil.cpu_count(),
+            'cpu_count': psutil.cpu_count(),
+            'cpu_cur_freq': round(psutil.cpu_freq()[0], 2),
+            'cpu_max_freq': psutil.cpu_freq()[2],
+            'mem_percent': psutil.virtual_memory()[2],
+            'mem_usage': helper.human_readable_file_size(psutil.virtual_memory()[3]),
+            'mem_total': helper.human_readable_file_size(psutil.virtual_memory()[0]),
+            'disk_data': self._all_disk_usage()
+        }
+        server_stats = self.get_servers_stats()
+        data['servers'] = server_stats
+        data['node_stats'] = node_stats
+
+        return data
+
+    @staticmethod
+    def _get_process_stats(process_pid: int):
+        if process_pid is None:
+            process_stats = {
+                'cpu_usage': 0,
+                'memory_usage': 0,
+            }
+            return process_stats
+
+        try:
+            p = psutil.Process(process_pid)
+            dummy = p.cpu_percent()
+
+            # call it first so we can be more accurate per the docs
+            # https://giamptest.readthedocs.io/en/latest/#psutil.Process.cpu_percent
+
+            real_cpu = round(p.cpu_percent(interval=0.5) / psutil.cpu_count(), 2)
+
+            process_start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(p.create_time()))
+
+            # this is a faster way of getting data for a process
+            with p.oneshot():
+                process_stats = {
+                    'cpu_usage': real_cpu,
+                    'memory_usage': helper.human_readable_file_size(p.memory_info()[0]),
+                }
+            return process_stats
+
+        except Exception as e:
+            logger.error("Unable to get process details for pid: {} due to error: {}".format(process_pid, e))
+
+            # Dummy Data
+            process_stats = {
+                'cpu_usage': 0,
+                'memory_usage': 0,
+            }
+            return process_stats
+
+    # shamelessly stolen from https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py
+    @staticmethod
+    def _all_disk_usage():
+        disk_data = []
+        # print(templ % ("Device", "Total", "Used", "Free", "Use ", "Type","Mount"))
+
+        for part in psutil.disk_partitions(all=False):
+            if os.name == 'nt':
+                if 'cdrom' in part.opts or part.fstype == '':
+                    # skip cd-rom drives with no disk in it; they may raise
+                    # ENOENT, pop-up a Windows GUI error for a non-ready
+                    # partition or just hang.
+                    continue
+            usage = psutil.disk_usage(part.mountpoint)
+            disk_data.append(
+                {
+                    'device': part.device,
+                    'total': helper.human_readable_file_size(usage.total),
+                    'used': helper.human_readable_file_size(usage.used),
+                    'free': helper.human_readable_file_size(usage.free),
+                    'percent_used': int(usage.percent),
+                    'fs': part.fstype,
+                    'mount': part.mountpoint
+                }
+            )
+
+        return disk_data
+
+    @staticmethod
+    def get_world_size(world_path):
+
+        total_size = 0
+
+        # do a scan of the directories in the server path.
+        for root, dirs, files in os.walk(world_path, topdown=False):
+
+            # for each directory we find
+            for name in dirs:
+
+                # if the directory name is "region"
+                if name == "region":
+                    # log it!
+                    logger.debug("Path %s is called region. Getting directory size", os.path.join(root, name))
+
+                    # get this directory size, and add it to the total we have running.
+                    total_size += helper.get_dir_size(os.path.join(root, name))
+
+        level_total_size = helper.human_readable_file_size(total_size)
+
+        return level_total_size
+
+    @staticmethod
+    def parse_server_ping(ping_obj: object):
+        online_stats = {}
+
+        try:
+            online_stats = json.loads(ping_obj.players)
+
+        except Exception as e:
+            logger.info("Unable to read json from ping_obj: {}".format(e))
+            pass
+
+        ping_data = {
+            'online': online_stats.get("online", 0),
+            'max': online_stats.get('max', 0),
+            'players': online_stats.get('players', 0),
+            'server_description': ping_obj.description,
+            'server_version': ping_obj.version
+        }
+
+        return ping_data
+
+    def get_servers_stats(self):
+
+        server_stats_list = []
+        server_stats = {}
+
+        servers = controller.servers_list
+
+        logger.info("Getting Stats for all servers...")
+
+        for s in servers:
+
+            server_id = s.get('server_id', None)
+
+            logger.info('Getting stats for server: {}'.format(server_id))
+
+            # get our server object, settings and data dictionaries
+            server_obj = s.get('server_obj', None)
+            server_settings = s.get('server_settings', {})
+            server_data = s.get('server_data_obj', {})
+
+            # world data
+            world_name = server_settings.get('level-name', 'Unknown')
+            world_path = os.path.join(server_data.get('path', None), world_name)
+
+            # process stats
+            p_stats = self._get_process_stats(server_obj.PID)
+
+            # TODO: search server properties file for possible override of 127.0.0.1
+            internal_ip = server_data.get('server_ip', "127.0.0.1")
+            server_port = server_settings.get('server_port', "25565")
+
+            logger.debug("Pinging %s on port %s", internal_ip, server_port)
+            int_mc_ping = ping(internal_ip, int(server_port))
+
+            int_data = "Unable to connect"
+
+            if int_mc_ping:
+                int_data = self.parse_server_ping(int_mc_ping)
+
+            server_stats = {
+                'id': server_id,
+                'started': server_obj.get_start_time(),
+                'running': server_obj.check_running(),
+                'cpu': p_stats.get('cpu_usage', '0'),
+                'mem': p_stats.get('memory_usage', '0'),
+                'world_name': world_name,
+                'world_size': self.get_world_size(world_path),
+                'server_port': s['server_settings']['server-port'],
+                'int_ping_results': int_data
+            }
+
+            # add this servers data to the stack
+            server_stats_list.append(server_stats)
+
+
+        return server_stats_list
+
+    # todo: add stats to db
+    def record_stats(self):
+        stats_to_send = self.get_node_stats()
+        node_stats = stats_to_send.get('node_stats')
+        Host_Stats.insert({
+            Host_Stats.boot_time: node_stats.get('boot_time', "Unknown"),
+            Host_Stats.cpu_usage: round(node_stats.get('cpu_usage', 0), 2),
+            Host_Stats.cpu_cores: node_stats.get('cpu_count', 0),
+            Host_Stats.cpu_cur_freq: node_stats.get('cpu_cur_freq', 0),
+            Host_Stats.cpu_max_freq: node_stats.get('cpu_max_freq', 0),
+            Host_Stats.mem_usage: node_stats.get('mem_usage', "0 MB"),
+            Host_Stats.mem_percent: node_stats.get('mem_percent', 0),
+            Host_Stats.mem_total: node_stats.get('mem_total', "0 MB"),
+            Host_Stats.disk_json: node_stats.get('disk_data', '{}')
+        }).execute()
+
+        # delete 1 week old data
+        max_age = int(helper.get_setting("CRAFTY", "history_max_age"))
+        now = datetime.datetime.now()
+        last_week = now.day - max_age
+        Host_Stats.delete().where(Host_Stats.time < last_week).execute()
+
+stats = Stats()

app/classes/shared/models.py

@@ -54,9 +54,7 @@ class Host_Stats(BaseModel):
     mem_percent = FloatField(default=0)
     mem_usage = CharField(default="")
     mem_total = CharField(default="")
-    disk_percent = FloatField(default=0)
-    disk_usage = CharField(default="")
-    disk_total = CharField(default="")
+    disk_json = TextField(default="")
 
     class Meta:
         table_name = "host_stats"
@@ -111,7 +109,7 @@ class db_builder:
                 Users,
                 Host_Stats,
                 Webhooks,
-                Servers
+                Servers,
             ])
 
     @staticmethod
@@ -146,6 +144,10 @@ class db_shortcuts:
         query = Servers.select()
         return self.return_rows(query)
 
+    def get_latest_hosts_stats(self):
+        query = Host_Stats.select().order_by(Host_Stats.id.desc()).get()
+        return model_to_dict(query)
+
 
 installer = db_builder()
 db_helper = db_shortcuts()

app/classes/shared/tasks.py

@@ -9,6 +9,8 @@ from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
 from app.classes.web.tornado import webserver
 from app.classes.minecraft import server_props
+from app.classes.minecraft.stats import stats
+from app.classes.minecraft.controller import controller
 
 logger = logging.getLogger(__name__)
 
@@ -45,15 +47,16 @@ class TasksManager:
             time.sleep(5)
 
     def _main_graceful_exit(self):
-        # commander.stop_all_servers()
-        logger.info("***** Crafty Shutting Down *****\n\n")
-        console.info("***** Crafty Shutting Down *****\n\n")
         try:
             os.remove(helper.session_file)
             os.remove(os.path.join(helper.root_dir, 'exit.txt'))
             os.remove(os.path.join(helper.root_dir, '.header'))
+            controller.stop_all_servers()
         except:
             pass
+
+        logger.info("***** Crafty Shutting Down *****\n\n")
+        console.info("***** Crafty Shutting Down *****\n\n")
         self.main_thread_exiting = True
 
     def start_webserver(self):
@@ -80,6 +83,12 @@ class TasksManager:
             schedule.run_pending()
             time.sleep(1)
 
+    @staticmethod
+    def start_stats_recording():
+        stats_update_frequency = int(helper.get_setting("CRAFTY", 'stats_update_frequency'))
+        logger.info("Stats collection frequency set to {stats} seconds".format(stats=stats_update_frequency))
+        console.info("Stats collection frequency set to {stats} seconds".format(stats=stats_update_frequency))
+        schedule.every(stats_update_frequency).seconds.do(stats.record_stats)
 
 
 tasks_manager = TasksManager()

app/classes/web/panel_handler.py

@@ -8,6 +8,8 @@ from app.classes.shared.console import console
 from app.classes.shared.models import Users, installer
 from app.classes.web.base_handler import BaseHandler
 from app.classes.minecraft.controller import controller
+from app.classes.shared.models import db_helper
+
 
 logger = logging.getLogger(__name__)
 
@@ -21,12 +23,20 @@ class PanelHandler(BaseHandler):
 
         template = "panel/denied.html"
 
+
         page_data = {
             'version_data': "version_data_here",
-            'user_data': user_data
-        }
+            'user_data': user_data,
+            'server_stats': {
+                'total': len(controller.list_defined_servers()),
+                'running': len(controller.list_running_servers()),
+                'stopped': (len(controller.list_defined_servers()) - len(controller.list_running_servers()))
+            },
+            'hosts_data': db_helper.get_latest_hosts_stats()
+
+        }
+        print(page_data['hosts_data'])
 
-        servers = controller.list_defined_servers()
 
         if page == 'unauthorized':
             template = "panel/denied.html"

app/frontend/templates/base.html

@@ -30,9 +30,9 @@
       <!-- partial:partials/_navbar.html -->
       <nav class="navbar default-layout col-lg-12 col-12 p-0 fixed-top d-flex flex-row">
         <div class="text-center navbar-brand-wrapper d-flex align-items-top justify-content-center">
-          <a class="navbar-brand brand-logo" href="/pro/dashboard">
+          <a class="navbar-brand brand-logo" href="/panel/dashboard">
             <img src="/static/assets/images/logo_long.jpg" alt="logo" /> </a>
-          <a class="navbar-brand brand-logo-mini" href="/pro/dashboard">
+          <a class="navbar-brand brand-logo-mini" href="/panel/dashboard">
             <img src="/static/assets/images/logo_square.jpg" alt="logo" /> </a>
         </div>
         <div class="navbar-menu-wrapper d-flex align-items-center">
@@ -92,6 +92,8 @@
     <script src="/static/assets/vendors/js/vendor.bundle.base.js"></script>
     <!-- endinject -->
     <!-- Plugin js for this page -->
+    <script src="/static/assets/js/shared/off-canvas.js"></script>
+    <script src="/static/assets/js/shared/hoverable-collapse.js"></script>
 <!--    <script src="/static/assets/vendors/chart.js/Chart.min.js"></script>-->
 <!--    <script src="/static/assets/vendors/jvectormap/jquery-jvectormap.min.js"></script>-->
 <!--    <script src="/static/assets/vendors/jvectormap/jquery-jvectormap-world-mill-en.js"></script>-->
@@ -100,14 +102,14 @@
     <!-- End plugin js for this page -->
     <!-- inject:js -->
 
-    <script src="/static/assets/js/shared/hoverable-collapse.js"></script>
+<!--    <script src="/static/assets/js/shared/hoverable-collapse.js"></script>-->
     <script src="/static/assets/js/shared/misc.js"></script>
-    <script src="/static/assets/js/shared/settings.js"></script>
+<!--    <script src="/static/assets/js/shared/settings.js"></script>-->
 <!--    <script src="/static/assets/js/shared/todolist.js"></script>-->
     <script src="https://kit.fontawesome.com/b539899a58.js" crossorigin="anonymous"></script>
     <!-- endinject -->
     <!-- Custom js for this page -->
-<!--    <script src="/static/assets/js/demo_1/dashboard.js"></script>-->
+    <script src="/static/assets/js/demo_3/dashboard_2.js"></script>
     <!-- End custom js for this page -->
   </body>
 </html>

app/frontend/templates/panel/dashboard.html

@@ -31,8 +31,8 @@
 
                 </div>
                 <div class="wrapper my-auto ml-auto ml-lg-4">
-                  <p class="mb-0 text-success">3.5% CPU</p>
-                  <p class="mb-0 text-danger">80% Memory</p>
+                  <p class="mb-0 text-success">{{ data.get('hosts_data').get('cpu_usage') }} {{ _('CPU Usage') }}</p>
+                  <p class="mb-0 text-danger">{{ data.get('hosts_data').get('mem_percent') }}% {{ _('Memory Usage') }}</p>
                 </div>
               </div>
             </div>
@@ -40,13 +40,12 @@
               <div class="d-flex">
                 <div class="wrapper">
                   <h5 class="mb-1 font-weight-medium text-primary">Servers</h5>
-                  <h3 class="mb-0 font-weight-semibold">5</h3>
+                  <h3 class="mb-0 font-weight-semibold">{{ data['server_stats']['total'] }}</h3>
 
                 </div>
                 <div class="wrapper my-auto ml-auto ml-lg-4">
-                  <p class="mb-0 text-success">3 Online</p>
-                  <p class="mb-0 text-warning">1 Shutdown</p>
-                  <p class="mb-0 text-danger">1 Crashed</p>
+                  <p class="mb-0 text-success">{{ data['server_stats']['total'] }} {{_('Online')}}</p>
+                  <p class="mb-0 text-warning"> {{ data['server_stats']['running'] }} {{_('Shutdown')}}</p>
                 </div>
               </div>
             </div>

config.ini

@@ -7,3 +7,8 @@ language = en_EN
 cookie_expire = 30
 cookie_secret = random
 show_errors = true
+
+[CRAFTY]
+# max history in days
+history_max_age = 7
+stats_update_frequency = 10

main.py

@@ -12,6 +12,7 @@ from app.classes.shared.models import installer
 from app.classes.shared.tasks import tasks_manager
 from app.classes.minecraft.controller import controller
 
+
 def do_intro():
     logger.info("***** Crafty Controller Started *****")
 
@@ -56,7 +57,7 @@ if __name__ == '__main__':
 
     parser.add_argument('-i', '--ignore',
                         action='store_true',
-                        help="Ignore session.json files"
+                        help="Ignore session.lock files"
                         )
 
     parser.add_argument('-v', '--verbose',
@@ -85,6 +86,9 @@ if __name__ == '__main__':
     # slowing down reporting just for a 1/2 second so messages look cleaner
     time.sleep(.5)
 
+    # start stats logging
+    tasks_manager.start_stats_recording()
+
     # this should always be last
     tasks_manager.start_main_kill_switch_watcher()
 
@@ -93,6 +97,8 @@ if __name__ == '__main__':
         installer.create_tables()
         installer.default_settings()
 
+    # installer.create_tables()
+
     # init servers
     logger.info("Initializing all servers defined")
     console.info("Initializing all servers defined")