Mirrored_Repos/crafty-4
1
0
Fork 0
mirror of https://gitlab.com/crafty-controller/crafty-4.git synced 2024-08-30 18:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into dev-StatsDBImprovement

Browse Source
This commit is contained in:
Silversthorn 2022-06-01 23:52:49 +02:00
commit 9e4c8e99b1
19 changed files with 242 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/classes/minecraft/bedrock_ping.py
View File

@ -1,7 +1,12 @@
from contextlib import redirect_stderr
import os
import socket
import time
import psutil
from app.classes.shared.null_writer import NullWriter
with redirect_stderr(NullWriter()):
import psutil
class BedrockPing:

90
app/classes/minecraft/stats.py
View File

@ -1,16 +1,20 @@
from __future__ import annotations
from contextlib import redirect_stderr
import json
import logging
import datetime
import base64
import typing as t
import psutil
from app.classes.shared.null_writer import NullWriter
from app.classes.minecraft.mc_ping import ping
from app.classes.models.management import HostStats
from app.classes.models.servers import HelperServers
from app.classes.shared.helpers import Helpers
with redirect_stderr(NullWriter()):
import psutil
if t.TYPE_CHECKING:
from app.classes.shared.main_controller import Controller
@ -52,30 +56,66 @@ class Stats:
helper: Helpers
controller: Controller
@staticmethod
def try_get_boot_time():
try:
return datetime.datetime.fromtimestamp(
psutil.boot_time(), datetime.timezone.utc
)
except Exception as e:
logger.debug(f"error while getting boot time due to {e}")
# unix epoch with no timezone data
return datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
@staticmethod
def try_get_cpu_usage():
try:
return psutil.cpu_percent(interval=0.5) / psutil.cpu_count()
except Exception as e:
logger.debug(f"error while getting cpu percentage due to {e}")
return -1
def __init__(self, helper, controller):
self.helper = helper
self.controller = controller
def get_node_stats(self) -> NodeStatsReturnDict:
boot_time = datetime.datetime.fromtimestamp(psutil.boot_time())
try:
cpu_freq = psutil.cpu_freq()
except NotImplementedError:
cpu_freq = psutil._common.scpufreq(current=0, min=0, max=0)
memory = psutil.virtual_memory()
node_stats: NodeStatsDict = {
"boot_time": str(boot_time),
"cpu_usage": psutil.cpu_percent(interval=0.5) / psutil.cpu_count(),
"cpu_count": psutil.cpu_count(),
"cpu_cur_freq": round(cpu_freq[0], 2),
"cpu_max_freq": cpu_freq[2],
"mem_percent": memory.percent,
"mem_usage_raw": memory.used,
"mem_usage": Helpers.human_readable_file_size(memory.used),
"mem_total_raw": memory.total,
"mem_total": Helpers.human_readable_file_size(memory.total),
"disk_data": self._all_disk_usage(),
}
try:
node_stats: NodeStatsDict = {
"boot_time": str(Stats.try_get_boot_time()),
"cpu_usage": Stats.try_get_cpu_usage(),
"cpu_count": psutil.cpu_count(),
"cpu_cur_freq": round(cpu_freq[0], 2),
"cpu_max_freq": cpu_freq[2],
"mem_percent": memory.percent,
"mem_usage_raw": memory.used,
"mem_usage": Helpers.human_readable_file_size(memory.used),
"mem_total_raw": memory.total,
"mem_total": Helpers.human_readable_file_size(memory.total),
"disk_data": Stats._try_all_disk_usage(),
}
except Exception as e:
logger.debug(f"error while getting host stats due to {e}")
node_stats: NodeStatsDict = {
"boot_time": str(
datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
),
"cpu_usage": -1,
"cpu_count": -1,
"cpu_cur_freq": -1,
"cpu_max_freq": -1,
"mem_percent": -1,
"mem_usage_raw": -1,
"mem_usage": "",
"mem_total_raw": -1,
"mem_total": "",
"disk_data": [],
}
# server_stats = self.get_servers_stats()
# data['servers'] = server_stats
@ -83,6 +123,14 @@ class Stats:
"node_stats": node_stats,
}
@staticmethod
def _try_get_process_stats(process):
try:
return Stats._get_process_stats(process)
except Exception as e:
logger.debug(f"error while getting process stats due to {e}")
return {"cpu_usage": -1, "memory_usage": -1, "mem_percentage": -1}
@staticmethod
def _get_process_stats(process):
if process is None:
@ -122,6 +170,14 @@ class Stats:
}
return process_stats
@staticmethod
def _try_all_disk_usage():
try:
return Stats._all_disk_usage()
except Exception as e:
logger.debug(f"error while getting disk data due to {e}")
return []
# Source: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py
@staticmethod
def _all_disk_usage() -> t.List[DiskDataDict]:
@ -258,6 +314,6 @@ class Stats:
# delete old data
max_age = self.helper.get_setting("history_max_age")
now = datetime.datetime.now()
last_week = now.day - max_age
minimum_to_exist = now - datetime.timedelta(days=max_age)
HostStats.delete().where(HostStats.time < last_week).execute()
HostStats.delete().where(HostStats.time < minimum_to_exist).execute()

2
app/classes/models/management.py
View File

@ -350,7 +350,7 @@ class HelpersManagement:
compress: bool = False,
):
logger.debug(f"Updating server {server_id} backup config with {locals()}")
if Backups.select().where(Backups.server_id == server_id).count() != 0:
if Backups.select().where(Backups.server_id == server_id).exists():
new_row = False
conf = {}
else:

2
app/classes/models/roles.py
View File

@ -89,4 +89,4 @@ class HelperRoles:
@staticmethod
def role_id_exists(role_id) -> bool:
return Roles.select().where(Roles.role_id == role_id).count() != 0
return Roles.select().where(Roles.role_id == role_id).exists()

36
app/classes/models/server_stats.py
View File

@ -137,8 +137,8 @@ class HelperServerStats:
)
return server_data
def insert_server_stats(self, server):
server_id = server.get("id", 0)
def insert_server_stats(self, server_stats):
server_id = server_stats.get("id", 0)
if server_id == 0:
logger.warning("Stats saving failed with error: Server unknown (id = 0)")
@ -146,21 +146,23 @@ class HelperServerStats:
ServerStats.insert(
{
ServerStats.server_id: server.get("id", 0),
ServerStats.started: server.get("started", ""),
ServerStats.running: server.get("running", False),
ServerStats.cpu: server.get("cpu", 0),
ServerStats.mem: server.get("mem", 0),
ServerStats.mem_percent: server.get("mem_percent", 0),
ServerStats.world_name: server.get("world_name", ""),
ServerStats.world_size: server.get("world_size", ""),
ServerStats.server_port: server.get("server_port", 0),
ServerStats.int_ping_results: server.get("int_ping_results", False),
ServerStats.online: server.get("online", False),
ServerStats.max: server.get("max", False),
ServerStats.players: server.get("players", False),
ServerStats.desc: server.get("desc", False),
ServerStats.version: server.get("version", False),
ServerStats.server_id: server_stats.get("id", 0),
ServerStats.started: server_stats.get("started", ""),
ServerStats.running: server_stats.get("running", False),
ServerStats.cpu: server_stats.get("cpu", 0),
ServerStats.mem: server_stats.get("mem", 0),
ServerStats.mem_percent: server_stats.get("mem_percent", 0),
ServerStats.world_name: server_stats.get("world_name", ""),
ServerStats.world_size: server_stats.get("world_size", ""),
ServerStats.server_port: server_stats.get("server_port", 0),
ServerStats.int_ping_results: server_stats.get(
"int_ping_results", False
),
ServerStats.online: server_stats.get("online", False),
ServerStats.max: server_stats.get("max", False),
ServerStats.players: server_stats.get("players", False),
ServerStats.desc: server_stats.get("desc", False),
ServerStats.version: server_stats.get("version", False),
}
).execute(self.database)

2
app/classes/models/users.py
View File

@ -275,7 +275,7 @@ class HelperUsers:
@staticmethod
def user_id_exists(user_id):
return Users.select().where(Users.user_id == user_id).count() != 0
return Users.select().where(Users.user_id == user_id).exists()
# **********************************************************************************
# User_Roles Methods

6
app/classes/shared/command.py
View File

@ -123,3 +123,9 @@ class MainPrompt(cmd.Cmd):
def help_import3(self):
Console.help("Import users and servers from Crafty 3")
def help_set_passwd(self):
Console.help("Set a user's password. Example: set_passwd admin")
def help_threads(self):
Console.help("Get all of the Python threads used by Crafty")

32
app/classes/shared/helpers.py
View File

@ -16,15 +16,18 @@ import pathlib
import ctypes
from datetime import datetime
from socket import gethostname
from contextlib import suppress
import psutil
from contextlib import redirect_stderr, suppress
from app.classes.shared.null_writer import NullWriter
from app.classes.shared.console import Console
from app.classes.shared.installer import installer
from app.classes.shared.file_helpers import FileHelpers
from app.classes.shared.translation import Translation
from app.classes.web.websocket_helper import WebSocketHelper
with redirect_stderr(NullWriter()):
import psutil
logger = logging.getLogger(__name__)
try:
@ -194,7 +197,6 @@ class Helpers:
return cmd_out
def get_setting(self, key, default_return=False):
try:
with open(self.settings_file, "r", encoding="utf-8") as f:
data = json.load(f)
@ -202,10 +204,8 @@ class Helpers:
if key in data.keys():
return data.get(key)
else:
logger.error(f"Config File Error: setting {key} does not exist")
Console.error(f"Config File Error: setting {key} does not exist")
return default_return
logger.error(f'Config File Error: Setting "{key}" does not exist')
Console.error(f'Config File Error: Setting "{key}" does not exist')
except Exception as e:
logger.critical(
@ -217,22 +217,19 @@ class Helpers:
return default_return
def set_setting(self, key, new_value, default_return=False):
def set_setting(self, key, new_value):
try:
with open(self.settings_file, "r", encoding="utf-8") as f:
data = json.load(f)
if key in data.keys():
data[key] = new_value
with open(self.settings_file, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
return True
else:
logger.error(f"Config File Error: setting {key} does not exist")
Console.error(f"Config File Error: setting {key} does not exist")
return default_return
with open(self.settings_file, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
logger.error(f'Config File Error: Setting "{key}" does not exist')
Console.error(f'Config File Error: Setting "{key}" does not exist')
except Exception as e:
logger.critical(
@ -241,6 +238,7 @@ class Helpers:
Console.critical(
f"Config File Error: Unable to read {self.settings_file} due to {e}"
)
return False
@staticmethod
def get_local_ip():
@ -690,6 +688,8 @@ class Helpers:
# directory already exists - non-blocking error
except FileExistsError:
pass
except PermissionError as e:
logger.critical(f"Check generated exception due to permssion error: {e}")
def create_self_signed_cert(self, cert_dir=None):

12
app/classes/shared/null_writer.py Normal file
View File

@ -0,0 +1,12 @@
import logging
import os
logger = logging.getLogger(__name__)
class NullWriter:
def write(self, data):
if os.environ.get("CRAFTY_LOG_NULLWRITER", "false") == "true":
logger.debug(data)
if os.environ.get("CRAFTY_PRINT_NULLWRITER", "false") == "true":
print(data)

26
app/classes/shared/server.py
View File

@ -1,3 +1,4 @@
from contextlib import redirect_stderr
import os
import re
import time
@ -8,8 +9,6 @@ import logging.config
import subprocess
import html
import tempfile
import psutil
from psutil import NoSuchProcess
# TZLocal is set as a hidden import on win pipeline
from tzlocal import get_localzone
@ -25,6 +24,11 @@ from app.classes.models.server_permissions import PermissionsServers
from app.classes.shared.console import Console
from app.classes.shared.helpers import Helpers
from app.classes.shared.file_helpers import FileHelpers
from app.classes.shared.null_writer import NullWriter
with redirect_stderr(NullWriter()):
import psutil
from psutil import NoSuchProcess
logger = logging.getLogger(__name__)
@ -632,6 +636,7 @@ class ServerInstance:
# send it
self.process.stdin.write(f"{command}\n".encode("utf-8"))
self.process.stdin.flush()
return True
def crash_detected(self, name):
@ -883,7 +888,6 @@ class ServerInstance:
os.remove(Helpers.get_os_understandable_path(oldfile_path))
self.is_backingup = False
FileHelpers.del_dirs(temp_dir)
logger.info(f"Backup of server: {self.name} completed")
self.server_scheduler.remove_job("backup_" + str(self.server_id))
results = {"percent": 100, "total_files": 0, "current_file": 0}
@ -906,7 +910,6 @@ class ServerInstance:
).format(self.name),
)
time.sleep(3)
return
except:
logger.exception(
f"Failed to create backup of server {self.name} (ID {self.server_id})"
@ -921,7 +924,8 @@ class ServerInstance:
results,
)
self.is_backingup = False
return
finally:
FileHelpers.del_dirs(temp_dir)
def backup_status(self, source_path, dest_path):
results = Helpers.calc_percent(source_path, dest_path)
@ -1206,7 +1210,7 @@ class ServerInstance:
server_path = server["path"]
# process stats
p_stats = Stats._get_process_stats(self.process)
p_stats = Stats._try_get_process_stats(self.process)
# TODO: search server properties file for possible override of 127.0.0.1
internal_ip = server["server_ip"]
@ -1339,7 +1343,7 @@ class ServerInstance:
server_path = server_dt["path"]
# process stats
p_stats = Stats._get_process_stats(self.process)
p_stats = Stats._try_get_process_stats(self.process)
# TODO: search server properties file for possible override of 127.0.0.1
# internal_ip = server['server_ip']
@ -1453,12 +1457,12 @@ class ServerInstance:
def record_server_stats(self):
server = self.get_servers_stats()
self.stats_helper.insert_server_stats(server)
server_stats = self.get_servers_stats()
self.stats_helper.insert_server_stats(server_stats)
# delete old data
max_age = self.helper.get_setting("history_max_age")
now = datetime.datetime.now()
last_week = now.day - max_age
minimum_to_exist = now - datetime.timedelta(days=max_age)
self.stats_helper.remove_old_stats(last_week)
self.stats_helper.remove_old_stats(minimum_to_exist)

1
app/classes/shared/tasks.py
View File

@ -127,6 +127,7 @@ class TasksManager:
svr.jar_update()
else:
svr.send_command(command)
HelpersManagement.mark_command_complete(cmd.command_id)
time.sleep(1)

9
app/classes/web/base_api_handler.py
View File

@ -19,5 +19,12 @@ class BaseApiHandler(BaseHandler):
delete = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
patch = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
put = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
options = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
# }}}
def options(self, *_, **__):
"""
Fix CORS
"""
# no body
self.set_status(204)
self.finish()

6
app/classes/web/routes/api/api_handlers.py
View File

@ -22,6 +22,7 @@ from app.classes.web.routes.api.servers.server.public import (
ApiServersServerPublicHandler,
)
from app.classes.web.routes.api.servers.server.stats import ApiServersServerStatsHandler
from app.classes.web.routes.api.servers.server.stdin import ApiServersServerStdinHandler
from app.classes.web.routes.api.servers.server.users import ApiServersServerUsersHandler
from app.classes.web.routes.api.users.index import ApiUsersIndexHandler
from app.classes.web.routes.api.users.user.index import ApiUsersUserIndexHandler
@ -127,6 +128,11 @@ def api_handlers(handler_args):
ApiServersServerPublicHandler,
handler_args,
),
(
r"/api/v2/servers/([0-9]+)/stdin/?",
ApiServersServerStdinHandler,
handler_args,
),
(
r"/api/v2/roles/?",
ApiRolesIndexHandler,

2
app/classes/web/routes/api/servers/server/action.py
View File

@ -43,7 +43,7 @@ class ApiServersServerActionHandler(BaseApiHandler):
def _clone_server(self, server_id, user_id):
def is_name_used(name):
return Servers.select().where(Servers.server_name == name).count() != 0
return Servers.select().where(Servers.server_name == name).exists()
server_data = self.controller.servers.get_server_data_by_id(server_id)
server_uuid = server_data.get("server_uuid")

9
app/classes/web/routes/api/servers/server/logs.py
View File

@ -8,6 +8,8 @@ from app.classes.web.base_api_handler import BaseApiHandler
logger = logging.getLogger(__name__)
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
class ApiServersServerLogsHandler(BaseApiHandler):
def get(self, server_id: str):
@ -45,6 +47,9 @@ class ApiServersServerLogsHandler(BaseApiHandler):
self.helper.get_os_understandable_path(server_data["log_path"]),
log_lines,
)
# Remove newline characters from the end of the lines
raw_lines = [line.rstrip("\r\n") for line in raw_lines]
else:
raw_lines = ServerOutBuf.lines.get(server_id, [])
@ -53,9 +58,7 @@ class ApiServersServerLogsHandler(BaseApiHandler):
for line in raw_lines:
try:
if not disable_ansi_strip:
line = re.sub(
"(\033\\[(0;)?[0-9]*[A-z]?(;[0-9])?m?)|(> )", "", line
)
line = ansi_escape.sub("", line)
line = re.sub("[A-z]{2}\b\b", "", line)
line = html.escape(line)

10
app/classes/web/routes/api/servers/server/stats.py
View File

@ -1,7 +1,6 @@
import logging
from playhouse.shortcuts import model_to_dict
from app.classes.models.server_stats import HelperServerStats
from app.classes.web.base_api_handler import BaseApiHandler
from app.classes.controllers.servers_controller import ServersController
logger = logging.getLogger(__name__)
@ -17,12 +16,13 @@ class ApiServersServerStatsHandler(BaseApiHandler):
# if the user doesn't have access to the server, return an error
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
srv = ServersController().get_server_instance_by_id(server_id)
latest = srv.stats_helper.get_latest_server_stats()
self.finish_json(
200,
{
"status": "ok",
"data": model_to_dict(
HelperServerStats.get_latest_server_stats(server_id)[0]
),
"data": latest,
},
)

47
app/classes/web/routes/api/servers/server/stdin.py Normal file
View File

@ -0,0 +1,47 @@
import logging
from app.classes.models.server_permissions import EnumPermissionsServer
from app.classes.web.base_api_handler import BaseApiHandler
logger = logging.getLogger(__name__)
class ApiServersServerStdinHandler(BaseApiHandler):
def post(self, server_id: str):
auth_data = self.authenticate_user()
if not auth_data:
return
if server_id not in [str(x["server_id"]) for x in auth_data[0]]:
# if the user doesn't have access to the server, return an error
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
if (
EnumPermissionsServer.COMMANDS
not in self.controller.server_perms.get_user_id_permissions_list(
auth_data[4]["user_id"], server_id
)
):
# if the user doesn't have Commands permission, return an error
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
svr = self.controller.get_server_obj_optional(server_id)
if svr is None:
# It's in auth_data[0] but not as a Server object
logger.critical(
"Something has gone VERY wrong! "
"Crafty can't access the server object. "
"Please report this to the devs"
)
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
if svr.send_command(self.request.body.decode("utf-8")):
return self.finish_json(
200,
{"status": "ok"},
)
self.finish_json(
200,
{"status": "error", "error": "SERVER_NOT_RUNNING"},
)

5
app/frontend/templates/base.html Normal file → Executable file
View File

@ -21,6 +21,11 @@
<!-- endinject -->
<!-- Plugin css for this page-->
<link rel="stylesheet" href="/static/assets/vendors/jvectormap/jquery-jvectormap.css">
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<!-- End Plugin css for this page-->
<!-- Layout styles -->
<link rel="stylesheet" href="/static/assets/css/dark/style.css">
<!-- End Layout styles -->

26
app/translations/fi_FI.json
View File

@ -65,7 +65,7 @@
"cannotSee": "Etkö näe kaikkea?",
"cannotSeeOnMobile": "Etkö näe kaikkea mobiililaitteella?",
"cannotSeeOnMobile2": "Yritä vierittää taulukkoa sivuttain.",
"clone": "Klooni",
"clone": "Kloonaa",
"cpuCores": "Suorittimen ytimet",
"cpuCurFreq": "Nykyinen kellotaajuus",
"cpuMaxFreq": "Maksimi kellotaajuus",
@ -91,10 +91,10 @@
"server": "Palvelin",
"servers": "Palvelimet",
"size": "Palvelimen hakemiston koko",
"start": "Alkaa",
"start": "Käynnistä",
"starting": "Myöhästynyt lähtö",
"status": "Tila",
"stop": "Lopettaa",
"stop": "Sammuta",
"version": "Versio",
"welcome": "Tervetuloa Crafty Controller"
},
@ -126,12 +126,12 @@
"print": "Tulosta"
},
"decimal": "",
"emptyTable": "Ei tietoja saatavilla taulukossa",
"info": "Näytetään _START_-_END_ / _TOTAL_ merkinnästä",
"infoEmpty": "Showing 0 to 0 of 0 entries",
"infoFiltered": "(filtered from _MAX_ total entries)",
"emptyTable": "Tietoja ei löytynyt",
"info": "Näytetään rivit _START_ - _END_ (yhteensä _TOTAL_ )",
"infoEmpty": "Näytetään 0 - 0 (yhteensä 0)",
"infoFiltered": "(suodatettu _MAX_ tuloksen joukosta)",
"infoPostFix": "",
"lengthMenu": "Show _MENU_ entries",
"lengthMenu": "Näytä kerralla _MENU_ riviä",
"loadingRecords": "Ladataan...",
"paginate": {
"first": "Ensimmäinen",
@ -411,8 +411,8 @@
"children": "Linkitetyt lapsitehtävät: ",
"command": "Komento",
"command-explain": "Minkä komennon haluat meidän suorittavan? Älä sisällytä '/'",
"cron": "Cron",
"cron-explain": "Kirjoita Cron-merkkijonosi",
"cron": "CRON",
"cron-explain": "Kirjoita CRON-ilmaisusi",
"custom": "Mukautettu komento",
"days": "Päivää",
"enabled": "Käytössä",
@ -421,15 +421,15 @@
"interval-explain": "Kuinka usein haluat tämän ajoituksen suoritettavan?",
"minutes": "Minuuttia",
"offset": "Viivepoikkeama",
"offset-explain": "Kuinka kauan meidän pitäisi odottaa tämän suorittamista ensimmäisen tehtävän suorittamisen jälkeen? (sekunteissa)",
"offset-explain": "Kuinka kauan Craftyn pitäisi odottaa tämän suorittamista ensimmäisen tehtävän suorittamisen jälkeen? (sekunteissa)",
"one-time": "Poista suorituksen jälkeen",
"parent": "Valitse isäntäajoitus",
"parent-explain": "Minkä ajoituksen pitäisi käynnistää tämä?",
"reaction": "Reaktio",
"reaction": "Ketjureaktio",
"restart": "Uudelleenkäynnistä palvelin",
"start": "Käynnistä palvelin",
"stop": "Sammuta palvelin",
"time": "Aika",
"time": "Päivänaika",
"time-explain": "Mihin aikaan haluat ajoituksen suoritettavan?"
},
"serverSchedules": {