complete rewrite of stats pulling, fixed default.json, changed the config file and handling

.gitignore

@@ -18,5 +18,6 @@ venv.bak/
 servers/
 session.lock
 .header
+default.json
 
 app/config/web

app/classes/minecraft/serverjars.py

@@ -2,6 +2,7 @@ import os
 import sys
 import json
 import time
+import shutil
 import logging
 from datetime import datetime
 
@@ -142,18 +143,17 @@ class ServerJars:
 
     def download_jar(self, server, version, path):
         base_url = "https://serverjars.com/api/fetchJar/{server}/{version}".format(server=server, version=version)
-        r = requests.get(base_url, timeout=2)
-        if r.status_code in [200, 201]:
 
+        # open a file stream
+        with requests.get(base_url, timeout=2, stream=True) as r:
             try:
-                with open(path, 'bw') as output:
+                with open(path, 'wb') as output:
-                    output.write(r.content)
+                    shutil.copyfileobj(r.raw, output)
-                    return True
+
             except Exception as e:
                 logger.error("Unable to save jar to {path} due to error:{error}".format(path=path, error=e))
                 pass
 
-        logger.error("Got {} code from download, escaping".format(r.status_code))
         return False
 
     # todo: import server
@@ -169,6 +169,8 @@ class ServerJars:
         # download the jar
         self.download_jar(server, version, full_jar_path)
 
+        time.sleep(3)
+
         # todo: verify the MD5
 
         # put data in the db

app/classes/minecraft/stats.py

@@ -232,7 +232,6 @@ class Stats:
         server_stats = stats_to_send.get('servers')
 
         for server in server_stats:
-
             Server_Stats.insert({
                 Server_Stats.server_id: server.get('id', 0),
                 Server_Stats.started: server.get('started', ""),
@@ -252,7 +251,7 @@ class Stats:
             }).execute()
 
         # delete 1 week old data
-        max_age = int(helper.get_setting("CRAFTY", "history_max_age"))
+        max_age = helper.get_setting("history_max_age")
         now = datetime.datetime.now()
         last_week = now.day - max_age
 

app/classes/shared/helpers.py

@@ -34,7 +34,7 @@ class Helpers:
         self.servers_dir = os.path.join(self.root_dir, 'servers')
 
         self.session_file = os.path.join(self.root_dir, 'session.lock')
-        self.settings_file = os.path.join(self.root_dir, 'config.ini')
+        self.settings_file = os.path.join(self.root_dir, 'config.json')
 
         self.db_path = os.path.join(self.root_dir, 'crafty.sqlite')
         self.serverjar_cache = os.path.join(self.config_dir, 'serverjars.json')
@@ -52,14 +52,19 @@ class Helpers:
         logger.error("{} does not exits".format(file))
         return False
 
-    def get_setting(self, section, key):
+    def get_setting(self, key):
 
         try:
-            our_config = configparser.ConfigParser()
+            with open(self.settings_file, "r") as f:
-            our_config.read(self.settings_file)
+                data = json.load(f)
 
-            if our_config.has_option(section, key):
+            if key in data.keys():
-                return our_config[section][key]
+                return data.get(key)
+
+            else:
+                logger.error("Config File Error: setting {} does not exist".format(key))
+                console.error("Config File Error: setting {} does not exist".format(key))
+                return False
 
         except Exception as e:
             logger.critical("Config File Error: Unable to read {} due to {}".format(self.settings_file, e))
@@ -366,6 +371,9 @@ class Helpers:
             with open(default_file, 'r') as f:
                 data = json.load(f)
 
+            del_json = helper.get_setting('delete_default_json')
+
+            if del_json:
                 os.remove(default_file)
 
         return data

app/classes/shared/models.py

@@ -196,8 +196,14 @@ class db_shortcuts:
         return self.return_rows(query)
 
     def get_all_servers_stats(self):
-        query = Server_Stats.select().order_by(Server_Stats.stats_id.desc()).limit(1)
+        servers = self.get_all_defined_servers()
-        return self.return_rows(query)
+        server_data = []
+
+        for s in servers:
+            latest = Server_Stats.select().where(Server_Stats.server_id == s.get('server_id')).order_by(Server_Stats.created.desc()).limit(1)
+            server_data.append({'server_data': s, "stats": self.return_rows(latest)})
+        # print(server_data)
+        return server_data
 
     def get_latest_hosts_stats(self):
         query = Host_Stats.select().order_by(Host_Stats.id.desc()).get()

app/classes/shared/tasks.py

@@ -86,7 +86,7 @@ class TasksManager:
 
     @staticmethod
     def start_stats_recording():
-        stats_update_frequency = int(helper.get_setting("CRAFTY", 'stats_update_frequency'))
+        stats_update_frequency = helper.get_setting('stats_update_frequency')
         logger.info("Stats collection frequency set to {stats} seconds".format(stats=stats_update_frequency))
         console.info("Stats collection frequency set to {stats} seconds".format(stats=stats_update_frequency))
 

app/classes/web/panel_handler.py

@@ -48,10 +48,10 @@ class PanelHandler(BaseHandler):
 
         elif page == 'dashboard':
             page_data['servers'] = db_helper.get_all_servers_stats()
+
             for s in page_data['servers']:
                 try:
                     data = json.loads(s['int_ping_results'])
-                    print(data)
                     s['int_ping_results'] = data
                 except:
                     pass

app/classes/web/public_handler.py

@@ -24,7 +24,7 @@ class PublicHandler(BaseHandler):
 
     def set_current_user(self, user):
 
-        expire_days = helper.get_setting("WEB", 'cookie_expire')
+        expire_days = helper.get_setting('cookie_expire')
 
         # if helper comes back with false
         if not expire_days:
@@ -111,6 +111,6 @@ class PublicHandler(BaseHandler):
 
                 next_page = "/panel/dashboard"
                 self.redirect(next_page)
-        else:
+
             self.redirect("/public/login")
 

app/classes/web/server_handler.py

@@ -7,6 +7,7 @@ from app.classes.web.base_handler import BaseHandler
 from app.classes.minecraft.controller import controller
 from app.classes.shared.models import db_helper, Servers
 from app.classes.minecraft.serverjars import server_jar_obj
+from app.classes.minecraft.stats import stats
 
 
 logger = logging.getLogger(__name__)
@@ -94,7 +95,9 @@ class ServerHandler(BaseHandler):
             server_parts = server.split("|")
 
             success = server_jar_obj.build_server(server_parts[0], server_parts[1], server_name, min_mem, max_mem, port)
+
             if success:
+                stats.record_stats()
                 self.redirect("/panel/dashboard")
 
         self.render(

app/classes/web/tornado.py

@@ -80,27 +80,24 @@ class webserver:
         # let's verify we have an SSL cert
         helper.create_self_signed_cert()
 
-        http_port = helper.get_setting("WEB", 'http_port')
+        http_port = helper.get_setting('http_port')
-        https_port = helper.get_setting("WEB", 'https_port')
+        https_port = helper.get_setting('https_port')
 
-        lang = helper.get_setting("WEB", 'language')
+        lang = helper.get_setting('language')
-        debug_errors = helper.get_setting("WEB", 'show_errors')
+        debug_errors = helper.get_setting('show_errors')
-        cookie_secret = helper.get_setting("WEB", 'cookie_secret')
+        cookie_secret = helper.get_setting('cookie_secret')
 
         if cookie_secret is False:
             cookie_secret = helper.random_string_generator(32)
 
-        if cookie_secret.lower == "random":
-            cookie_secret = helper.random_string_generator(32)
-
         if not lang:
             lang = "en_EN"
 
         if not http_port:
-            port = 8000
+            http_port = 8000
 
         if not https_port:
-            port = 8443
+            https_port = 8443
 
         cert_objects = {
             'certfile': os.path.join(helper.config_dir, 'web', 'certs', 'commander.cert.pem'),

app/config/serverjars.json

@@ -1,5 +1,5 @@
 {
-    "last_refreshed": "08/26/2020, 08:13:15",
+    "last_refreshed": "08/27/2020, 08:26:57",
     "servers": {
         "nukkitx": [
             "1.14"

app/frontend/templates/panel/dashboard.html

@@ -107,49 +107,49 @@
                 <tr>
                   <td>
                     <i class="fas fa-server"></i>
-                    <a href="/panel/server_detail?id={{s['server_id']}}">
+                    <a href="/panel/server_detail?id={{server['server_data']['server_id']}}">
-                      {{ server['server_id']['server_name'] }}
+                      {{ server['server_data']['server_name'] }}
                     </a>
                   </td>
 
                   <td>
 
-                    {% if server['running']%}
+                    {% if server['stats'][0]['running'] %}
-                      <a class="stop_button" data-id="{{server['server_id']['server_id']}}"> <i class="fas fa-stop"></i></a> &nbsp;
+                      <a class="stop_button" data-id="{{server['server_data']}}"> <i class="fas fa-stop"></i></a> &nbsp;
-                      <a class="restart_button" data-id="{{server['server_id']['server_id']}}"> <i class="fas fa-sync"></i></a> &nbsp;
+                      <a class="restart_button" data-id="{{server['server_data']}}"> <i class="fas fa-sync"></i></a> &nbsp;
                     {% else %}
-                      <a data-id="{{server['server_id']['server_id']}}" class="play_button"><i class="fas fa-play"></i></a> &nbsp;
+                      <a data-id="{{server['server_data']['server_id']}}" class="play_button"><i class="fas fa-play"></i></a> &nbsp;
                     {% end %}
                   </td>
 
                   <td>
-                    <div class="progress" data-toggle="tooltip" data-placement="top" title="{{server['cpu']}}">
+                    <div class="progress" data-toggle="tooltip" data-placement="top" title="{{server['stats'][0]['cpu']}}">
-                      <div class="progress-bar bg-success" role="progressbar" style="width: {{server['cpu']}}%" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"></div>
+                      <div class="progress-bar bg-success" role="progressbar" style="width: {{server['stats'][0]['cpu']}}%" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"></div>
 
                     </div>
-                    {{server['cpu']}}%
+                    {{server['stats'][0]['cpu']}}%
                   </td>
 
                   <td>
-                    <div class="progress" data-toggle="tooltip" data-placement="top" title="{{server['mem']}}">
+                    <div class="progress" data-toggle="tooltip" data-placement="top" title="{{server['stats'][0]['mem']}}">
-                      <div class="progress-bar bg-success" role="progressbar" style="width: {{server['mem_percent']}}%" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"></div>
+                      <div class="progress-bar bg-success" role="progressbar" style="width: {{server['stats'][0]['mem_percent']}}%" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"></div>
                     </div>
-                    {{server['mem_percent']}}% - {{server['mem']}}
+                    {{server['stats'][0]['mem_percent']}}% - {{server['stats'][0]['mem']}}
                   </td>
                   <td>
-                    {{ server['world_name'] }} : {{ server['world_size'] }}
+                    {{ server['stats'][0]['world_name'] }} : {{ server['stats'][0]['world_size'] }}
                   </td>
                   <td>
-                    {% if server['int_ping_results'] == True %}
+                    {% if server['stats'][0]['int_ping_results'] %}
-                      {{ server['online'] }} / {{ server['max'] }} Max<br />
+                      {{ server['stats'][0]['online'] }} / {{ server['stats'][0]['max'] }} Max<br />
-                      {{ server['desc'] }} <br />
+                      {{ server['stats'][0]['desc'] }} <br />
-                      {{ server['version'] }}
+                      {{ server['stats'][0]['version'] }}
                     {% else %}
                       {{ _("Unable to connect to server") }}
                     {% end %}
                   </td>
                   <td>
-                    {% if server['running'] %}
+                    {% if server['stats'][0]['running'] %}
                     <i class="fas fa-thumbs-up"></i> <span class="text-success">Online</span>
                     {% else %}
                     <i class="fas fa-thumbs-down"></i> <span class="text-danger">Offline</span>

app/frontend/templates/server/wizard.html

@@ -67,7 +67,7 @@
                   </div>
 
                 </div>
-                <button type="submit" class="btn btn-primary mr-2">Build My Server!</button>
+                <button type="submit" class="btn btn-primary mr-2" onclick="wait_msg()">Build My Server!</button>
                 <button type="reset" class="btn btn-danger mr-2">Reset Form</button>
 
               </form>
@@ -90,6 +90,13 @@
     });
   });
 
+  function wait_msg(){
+    bootbox.alert({
+        title: 'Downloading Server...',
+        message: '<i class="fas fa-cloud-download"></i> Please be patient as we download the server'
+      });
+  }
+
   function check_sizes(){
     max_mem = parseFloat($('#max_memory').val());
     min_mem = parseFloat($('#min_memory').val());

config.ini

@@ -1,14 +0,0 @@
-[WEB]
-https = true
-force_https = false
-http_port = 8000
-https_port = 8443
-language = en_EN
-cookie_expire = 30
-cookie_secret = random
-show_errors = true
-
-[CRAFTY]
-# max history in days
-history_max_age = 7
-stats_update_frequency = 10

config.json

@@ -0,0 +1,12 @@
+{
+  "https": true,
+  "http_port": 8000,
+  "https_port": 8443,
+  "language": "en_EN",
+  "cookie_expire": 30,
+  "cookie_secret": "random",
+  "show_errors": true,
+  "history_max_age": 7,
+  "stats_update_frequency": 5,
+  "delete_default_json": false
+}

default.json.example

@@ -1,4 +1,4 @@
 {
-	"username": "admin"
+	"username": "admin",
 	"password": "crafty"
 }