Mirrored_Repos/crafty-4
1
0
Fork 0
mirror of https://gitlab.com/crafty-controller/crafty-4.git synced 2024-08-30 18:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'experimental/lukas-codebase-improvements' into 'dev'

Browse Source 
Lukas's codebase improvements

See merge request crafty-controller/crafty-4!277
This commit is contained in:
Andrew 2022-06-01 20:22:47 +00:00
commit 9864cecff1
19 changed files with 224 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/classes/controllers/server_perms_controller.py
View File

@ -8,7 +8,6 @@ from app.classes.models.users import HelperUsers, ApiKeys
from app.classes.models.roles import HelperRoles
from app.classes.models.servers import HelperServers
from app.classes.models.server_stats import HelperServerStats
from app.classes.shared.main_models import DatabaseShortcuts
logger = logging.getLogger(__name__)
@ -107,11 +106,11 @@ class ServerPermsController:
)
for server in authorized_servers:
latest = HelperServerStats.get_latest_server_stats(server.get("server_id"))
latest = HelperServerStats.get_server_stats_by_id(server.get("server_id"))
server_data.append(
{
"server_data": server,
"stats": DatabaseShortcuts.return_rows(latest)[0],
"stats": latest,
}
)
return server_data

5
app/classes/controllers/servers_controller.py
View File

@ -12,7 +12,6 @@ from app.classes.models.server_permissions import (
EnumPermissionsServer,
)
from app.classes.shared.helpers import Helpers
from app.classes.shared.main_models import DatabaseShortcuts
logger = logging.getLogger(__name__)
@ -153,7 +152,7 @@ class ServersController:
)
for server in authorized_servers:
latest = HelperServerStats.get_latest_server_stats(server.get("server_id"))
latest = HelperServerStats.get_server_stats_by_id(server.get("server_id"))
key_permissions = PermissionsServers.get_api_key_permissions_list(
api_key, server.get("server_id")
)
@ -164,7 +163,7 @@ class ServersController:
server_data.append(
{
"server_data": server,
"stats": DatabaseShortcuts.return_rows(latest)[0],
"stats": latest,
"user_command_permission": user_command_permission,
}
)

7
app/classes/minecraft/bedrock_ping.py
View File

@ -1,7 +1,12 @@
from contextlib import redirect_stderr
import os
import socket
import time
import psutil
from app.classes.shared.null_writer import NullWriter
with redirect_stderr(NullWriter()):
import psutil
class BedrockPing:

90
app/classes/minecraft/stats.py
View File

@ -1,16 +1,20 @@
from __future__ import annotations
from contextlib import redirect_stderr
import json
import logging
import datetime
import base64
import typing as t
import psutil
from app.classes.shared.null_writer import NullWriter
from app.classes.minecraft.mc_ping import ping
from app.classes.models.management import HostStats
from app.classes.models.servers import HelperServers
from app.classes.shared.helpers import Helpers
with redirect_stderr(NullWriter()):
import psutil
if t.TYPE_CHECKING:
from app.classes.shared.main_controller import Controller
@ -52,30 +56,66 @@ class Stats:
helper: Helpers
controller: Controller
@staticmethod
def try_get_boot_time():
try:
return datetime.datetime.fromtimestamp(
psutil.boot_time(), datetime.timezone.utc
)
except Exception as e:
logger.debug(f"error while getting boot time due to {e}")
# unix epoch with no timezone data
return datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
@staticmethod
def try_get_cpu_usage():
try:
return psutil.cpu_percent(interval=0.5) / psutil.cpu_count()
except Exception as e:
logger.debug(f"error while getting cpu percentage due to {e}")
return -1
def __init__(self, helper, controller):
self.helper = helper
self.controller = controller
def get_node_stats(self) -> NodeStatsReturnDict:
boot_time = datetime.datetime.fromtimestamp(psutil.boot_time())
try:
cpu_freq = psutil.cpu_freq()
except NotImplementedError:
cpu_freq = psutil._common.scpufreq(current=0, min=0, max=0)
memory = psutil.virtual_memory()
node_stats: NodeStatsDict = {
"boot_time": str(boot_time),
"cpu_usage": psutil.cpu_percent(interval=0.5) / psutil.cpu_count(),
"cpu_count": psutil.cpu_count(),
"cpu_cur_freq": round(cpu_freq[0], 2),
"cpu_max_freq": cpu_freq[2],
"mem_percent": memory.percent,
"mem_usage_raw": memory.used,
"mem_usage": Helpers.human_readable_file_size(memory.used),
"mem_total_raw": memory.total,
"mem_total": Helpers.human_readable_file_size(memory.total),
"disk_data": self._all_disk_usage(),
}
try:
node_stats: NodeStatsDict = {
"boot_time": str(Stats.try_get_boot_time()),
"cpu_usage": Stats.try_get_cpu_usage(),
"cpu_count": psutil.cpu_count(),
"cpu_cur_freq": round(cpu_freq[0], 2),
"cpu_max_freq": cpu_freq[2],
"mem_percent": memory.percent,
"mem_usage_raw": memory.used,
"mem_usage": Helpers.human_readable_file_size(memory.used),
"mem_total_raw": memory.total,
"mem_total": Helpers.human_readable_file_size(memory.total),
"disk_data": Stats._try_all_disk_usage(),
}
except Exception as e:
logger.debug(f"error while getting host stats due to {e}")
node_stats: NodeStatsDict = {
"boot_time": str(
datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
),
"cpu_usage": -1,
"cpu_count": -1,
"cpu_cur_freq": -1,
"cpu_max_freq": -1,
"mem_percent": -1,
"mem_usage_raw": -1,
"mem_usage": "",
"mem_total_raw": -1,
"mem_total": "",
"disk_data": [],
}
# server_stats = self.get_servers_stats()
# data['servers'] = server_stats
@ -83,6 +123,14 @@ class Stats:
"node_stats": node_stats,
}
@staticmethod
def _try_get_process_stats(process):
try:
return Stats._get_process_stats(process)
except Exception as e:
logger.debug(f"error while getting process stats due to {e}")
return {"cpu_usage": -1, "memory_usage": -1, "mem_percentage": -1}
@staticmethod
def _get_process_stats(process):
if process is None:
@ -122,6 +170,14 @@ class Stats:
}
return process_stats
@staticmethod
def _try_all_disk_usage():
try:
return Stats._all_disk_usage()
except Exception as e:
logger.debug(f"error while getting disk data due to {e}")
return []
# Source: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py
@staticmethod
def _all_disk_usage() -> t.List[DiskDataDict]:
@ -258,6 +314,6 @@ class Stats:
# delete old data
max_age = self.helper.get_setting("history_max_age")
now = datetime.datetime.now()
last_week = now.day - max_age
minimum_to_exist = now - datetime.timedelta(days=max_age)
HostStats.delete().where(HostStats.time < last_week).execute()
HostStats.delete().where(HostStats.time < minimum_to_exist).execute()

2
app/classes/models/management.py
View File

@ -350,7 +350,7 @@ class HelpersManagement:
compress: bool = False,
):
logger.debug(f"Updating server {server_id} backup config with {locals()}")
if Backups.select().where(Backups.server_id == server_id).count() != 0:
if Backups.select().where(Backups.server_id == server_id).exists():
new_row = False
conf = {}
else:

2
app/classes/models/roles.py
View File

@ -89,4 +89,4 @@ class HelperRoles:
@staticmethod
def role_id_exists(role_id) -> bool:
return Roles.select().where(Roles.role_id == role_id).count() != 0
return Roles.select().where(Roles.role_id == role_id).exists()

42
app/classes/models/server_stats.py
View File

@ -143,8 +143,8 @@ class HelperServerStats:
return server_data
@staticmethod
def insert_server_stats(server):
server_id = server.get("id", 0)
def insert_server_stats(server_stats):
server_id = server_stats.get("id", 0)
database = HelperServerStats.select_database(server_id)
if server_id == 0:
@ -153,28 +153,32 @@ class HelperServerStats:
ServerStats.insert(
{
ServerStats.server_id: server.get("id", 0),
ServerStats.started: server.get("started", ""),
ServerStats.running: server.get("running", False),
ServerStats.cpu: server.get("cpu", 0),
ServerStats.mem: server.get("mem", 0),
ServerStats.mem_percent: server.get("mem_percent", 0),
ServerStats.world_name: server.get("world_name", ""),
ServerStats.world_size: server.get("world_size", ""),
ServerStats.server_port: server.get("server_port", 0),
ServerStats.int_ping_results: server.get("int_ping_results", False),
ServerStats.online: server.get("online", False),
ServerStats.max: server.get("max", False),
ServerStats.players: server.get("players", False),
ServerStats.desc: server.get("desc", False),
ServerStats.version: server.get("version", False),
ServerStats.server_id: server_stats.get("id", 0),
ServerStats.started: server_stats.get("started", ""),
ServerStats.running: server_stats.get("running", False),
ServerStats.cpu: server_stats.get("cpu", 0),
ServerStats.mem: server_stats.get("mem", 0),
ServerStats.mem_percent: server_stats.get("mem_percent", 0),
ServerStats.world_name: server_stats.get("world_name", ""),
ServerStats.world_size: server_stats.get("world_size", ""),
ServerStats.server_port: server_stats.get("server_port", 0),
ServerStats.int_ping_results: server_stats.get(
"int_ping_results", False
),
ServerStats.online: server_stats.get("online", False),
ServerStats.max: server_stats.get("max", False),
ServerStats.players: server_stats.get("players", False),
ServerStats.desc: server_stats.get("desc", False),
ServerStats.version: server_stats.get("version", False),
}
).execute(database)
@staticmethod
def remove_old_stats(server_id, last_week):
def remove_old_stats(server_id, minimum_to_exist):
database = HelperServerStats.select_database(server_id)
ServerStats.delete().where(ServerStats.created < last_week).execute(database)
ServerStats.delete().where(ServerStats.created < minimum_to_exist).execute(
database
)
@staticmethod
def get_latest_server_stats(server_id):

2
app/classes/models/users.py
View File

@ -275,7 +275,7 @@ class HelperUsers:
@staticmethod
def user_id_exists(user_id):
return Users.select().where(Users.user_id == user_id).count() != 0
return Users.select().where(Users.user_id == user_id).exists()
# **********************************************************************************
# User_Roles Methods

6
app/classes/shared/command.py
View File

@ -119,3 +119,9 @@ class MainPrompt(cmd.Cmd):
def help_import3(self):
Console.help("Import users and servers from Crafty 3")
def help_set_passwd(self):
Console.help("Set a user's password. Example: set_passwd admin")
def help_threads(self):
Console.help("Get all of the Python threads used by Crafty")

30
app/classes/shared/helpers.py
View File

@ -16,15 +16,18 @@ import pathlib
import ctypes
from datetime import datetime
from socket import gethostname
from contextlib import suppress
import psutil
from contextlib import redirect_stderr, suppress
from app.classes.shared.null_writer import NullWriter
from app.classes.shared.console import Console
from app.classes.shared.installer import installer
from app.classes.shared.file_helpers import FileHelpers
from app.classes.shared.translation import Translation
from app.classes.web.websocket_helper import WebSocketHelper
with redirect_stderr(NullWriter()):
import psutil
logger = logging.getLogger(__name__)
try:
@ -194,7 +197,6 @@ class Helpers:
return cmd_out
def get_setting(self, key, default_return=False):
try:
with open(self.settings_file, "r", encoding="utf-8") as f:
data = json.load(f)
@ -202,10 +204,8 @@ class Helpers:
if key in data.keys():
return data.get(key)
else:
logger.error(f"Config File Error: setting {key} does not exist")
Console.error(f"Config File Error: setting {key} does not exist")
return default_return
logger.error(f'Config File Error: Setting "{key}" does not exist')
Console.error(f'Config File Error: Setting "{key}" does not exist')
except Exception as e:
logger.critical(
@ -217,22 +217,19 @@ class Helpers:
return default_return
def set_setting(self, key, new_value, default_return=False):
def set_setting(self, key, new_value):
try:
with open(self.settings_file, "r", encoding="utf-8") as f:
data = json.load(f)
if key in data.keys():
data[key] = new_value
with open(self.settings_file, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
return True
else:
logger.error(f"Config File Error: setting {key} does not exist")
Console.error(f"Config File Error: setting {key} does not exist")
return default_return
with open(self.settings_file, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
logger.error(f'Config File Error: Setting "{key}" does not exist')
Console.error(f'Config File Error: Setting "{key}" does not exist')
except Exception as e:
logger.critical(
@ -241,6 +238,7 @@ class Helpers:
Console.critical(
f"Config File Error: Unable to read {self.settings_file} due to {e}"
)
return False
@staticmethod
def get_local_ip():

12
app/classes/shared/null_writer.py Normal file
View File

@ -0,0 +1,12 @@
import logging
import os
logger = logging.getLogger(__name__)
class NullWriter:
def write(self, data):
if os.environ.get("CRAFTY_LOG_NULLWRITER", "false") == "true":
logger.debug(data)
if os.environ.get("CRAFTY_PRINT_NULLWRITER", "false") == "true":
print(data)

21
app/classes/shared/server.py
View File

@ -1,3 +1,4 @@
from contextlib import redirect_stderr
import os
import re
import time
@ -8,8 +9,6 @@ import logging.config
import subprocess
import html
import tempfile
import psutil
from psutil import NoSuchProcess
# TZLocal is set as a hidden import on win pipeline
from tzlocal import get_localzone
@ -25,6 +24,11 @@ from app.classes.models.server_permissions import PermissionsServers
from app.classes.shared.console import Console
from app.classes.shared.helpers import Helpers
from app.classes.shared.file_helpers import FileHelpers
from app.classes.shared.null_writer import NullWriter
with redirect_stderr(NullWriter()):
import psutil
from psutil import NoSuchProcess
logger = logging.getLogger(__name__)
@ -626,6 +630,7 @@ class Server:
# send it
self.process.stdin.write(f"{command}\n".encode("utf-8"))
self.process.stdin.flush()
return True
def crash_detected(self, name):
@ -1199,7 +1204,7 @@ class Server:
server_path = server["path"]
# process stats
p_stats = Stats._get_process_stats(self.process)
p_stats = Stats._try_get_process_stats(self.process)
# TODO: search server properties file for possible override of 127.0.0.1
internal_ip = server["server_ip"]
@ -1332,7 +1337,7 @@ class Server:
server_path = server_dt["path"]
# process stats
p_stats = Stats._get_process_stats(self.process)
p_stats = Stats._try_get_process_stats(self.process)
# TODO: search server properties file for possible override of 127.0.0.1
# internal_ip = server['server_ip']
@ -1446,12 +1451,12 @@ class Server:
def record_server_stats(self):
server = self.get_servers_stats()
HelperServerStats.insert_server_stats(server)
server_stats = self.get_servers_stats()
HelperServerStats.insert_server_stats(server_stats)
# delete old data
max_age = self.helper.get_setting("history_max_age")
now = datetime.datetime.now()
last_week = now.day - max_age
minimum_to_exist = now - datetime.timedelta(days=max_age)
HelperServerStats.remove_old_stats(server.get("id", 0), last_week)
HelperServerStats.remove_old_stats(server_stats.get("id", 0), minimum_to_exist)

1
app/classes/shared/tasks.py
View File

@ -123,6 +123,7 @@ class TasksManager:
svr.jar_update()
else:
svr.send_command(command)
HelpersManagement.mark_command_complete(cmd.command_id)
time.sleep(1)

9
app/classes/web/base_api_handler.py
View File

@ -19,5 +19,12 @@ class BaseApiHandler(BaseHandler):
delete = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
patch = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
put = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
options = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]]
# }}}
def options(self, *_, **__):
"""
Fix CORS
"""
# no body
self.set_status(204)
self.finish()

6
app/classes/web/routes/api/api_handlers.py
View File

@ -22,6 +22,7 @@ from app.classes.web.routes.api.servers.server.public import (
ApiServersServerPublicHandler,
)
from app.classes.web.routes.api.servers.server.stats import ApiServersServerStatsHandler
from app.classes.web.routes.api.servers.server.stdin import ApiServersServerStdinHandler
from app.classes.web.routes.api.servers.server.users import ApiServersServerUsersHandler
from app.classes.web.routes.api.users.index import ApiUsersIndexHandler
from app.classes.web.routes.api.users.user.index import ApiUsersUserIndexHandler
@ -127,6 +128,11 @@ def api_handlers(handler_args):
ApiServersServerPublicHandler,
handler_args,
),
(
r"/api/v2/servers/([0-9]+)/stdin/?",
ApiServersServerStdinHandler,
handler_args,
),
(
r"/api/v2/roles/?",
ApiRolesIndexHandler,

2
app/classes/web/routes/api/servers/server/action.py
View File

@ -43,7 +43,7 @@ class ApiServersServerActionHandler(BaseApiHandler):
def _clone_server(self, server_id, user_id):
def is_name_used(name):
return Servers.select().where(Servers.server_name == name).count() != 0
return Servers.select().where(Servers.server_name == name).exists()
server_data = self.controller.servers.get_server_data_by_id(server_id)
server_uuid = server_data.get("server_uuid")

9
app/classes/web/routes/api/servers/server/logs.py
View File

@ -8,6 +8,8 @@ from app.classes.web.base_api_handler import BaseApiHandler
logger = logging.getLogger(__name__)
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
class ApiServersServerLogsHandler(BaseApiHandler):
def get(self, server_id: str):
@ -45,6 +47,9 @@ class ApiServersServerLogsHandler(BaseApiHandler):
self.helper.get_os_understandable_path(server_data["log_path"]),
log_lines,
)
# Remove newline characters from the end of the lines
raw_lines = [line.rstrip("\r\n") for line in raw_lines]
else:
raw_lines = ServerOutBuf.lines.get(server_id, [])
@ -53,9 +58,7 @@ class ApiServersServerLogsHandler(BaseApiHandler):
for line in raw_lines:
try:
if not disable_ansi_strip:
line = re.sub(
"(\033\\[(0;)?[0-9]*[A-z]?(;[0-9])?m?)|(> )", "", line
)
line = ansi_escape.sub("", line)
line = re.sub("[A-z]{2}\b\b", "", line)
line = html.escape(line)

5
app/classes/web/routes/api/servers/server/stats.py
View File

@ -1,5 +1,4 @@
import logging
from playhouse.shortcuts import model_to_dict
from app.classes.models.server_stats import HelperServerStats
from app.classes.web.base_api_handler import BaseApiHandler
@ -21,8 +20,6 @@ class ApiServersServerStatsHandler(BaseApiHandler):
200,
{
"status": "ok",
"data": model_to_dict(
HelperServerStats.get_latest_server_stats(server_id)[0]
),
"data": HelperServerStats.get_server_stats_by_id(server_id),
},
)

47
app/classes/web/routes/api/servers/server/stdin.py Normal file
View File

@ -0,0 +1,47 @@
import logging
from app.classes.models.server_permissions import EnumPermissionsServer
from app.classes.web.base_api_handler import BaseApiHandler
logger = logging.getLogger(__name__)
class ApiServersServerStdinHandler(BaseApiHandler):
def post(self, server_id: str):
auth_data = self.authenticate_user()
if not auth_data:
return
if server_id not in [str(x["server_id"]) for x in auth_data[0]]:
# if the user doesn't have access to the server, return an error
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
if (
EnumPermissionsServer.COMMANDS
not in self.controller.server_perms.get_user_id_permissions_list(
auth_data[4]["user_id"], server_id
)
):
# if the user doesn't have Commands permission, return an error
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
svr = self.controller.get_server_obj_optional(server_id)
if svr is None:
# It's in auth_data[0] but not as a Server object
logger.critical(
"Something has gone VERY wrong! "
"Crafty can't access the server object. "
"Please report this to the devs"
)
return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"})
if svr.send_command(self.request.body.decode("utf-8")):
return self.finish_json(
200,
{"status": "ok"},
)
self.finish_json(
200,
{"status": "error", "error": "SERVER_NOT_RUNNING"},
)