From 166272e12c9fa76cf93d68e166a17db59033c052 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 14:24:34 +0300 Subject: [PATCH 01/26] Replace `.count() != 0` with `.exists()` --- app/classes/models/management.py | 2 +- app/classes/models/roles.py | 2 +- app/classes/models/users.py | 2 +- app/classes/web/routes/api/servers/server/action.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/classes/models/management.py b/app/classes/models/management.py index d100ee31..6cdf7a8a 100644 --- a/app/classes/models/management.py +++ b/app/classes/models/management.py @@ -350,7 +350,7 @@ class HelpersManagement: compress: bool = False, ): logger.debug(f"Updating server {server_id} backup config with {locals()}") - if Backups.select().where(Backups.server_id == server_id).count() != 0: + if Backups.select().where(Backups.server_id == server_id).exists(): new_row = False conf = {} else: diff --git a/app/classes/models/roles.py b/app/classes/models/roles.py index 60338475..4d61e051 100644 --- a/app/classes/models/roles.py +++ b/app/classes/models/roles.py @@ -89,4 +89,4 @@ class HelperRoles: @staticmethod def role_id_exists(role_id) -> bool: - return Roles.select().where(Roles.role_id == role_id).count() != 0 + return Roles.select().where(Roles.role_id == role_id).exists() diff --git a/app/classes/models/users.py b/app/classes/models/users.py index 620a6688..b72f0530 100644 --- a/app/classes/models/users.py +++ b/app/classes/models/users.py @@ -275,7 +275,7 @@ class HelperUsers: @staticmethod def user_id_exists(user_id): - return Users.select().where(Users.user_id == user_id).count() != 0 + return Users.select().where(Users.user_id == user_id).exists() # ********************************************************************************** # User_Roles Methods diff --git a/app/classes/web/routes/api/servers/server/action.py b/app/classes/web/routes/api/servers/server/action.py index b8728b1e..d40fd199 100644 --- a/app/classes/web/routes/api/servers/server/action.py +++ b/app/classes/web/routes/api/servers/server/action.py @@ -43,7 +43,7 @@ class ApiServersServerActionHandler(BaseApiHandler): def _clone_server(self, server_id, user_id): def is_name_used(name): - return Servers.select().where(Servers.server_name == name).count() != 0 + return Servers.select().where(Servers.server_name == name).exists() server_data = self.controller.servers.get_server_data_by_id(server_id) server_uuid = server_data.get("server_uuid") From 4b707aa9d137b8457aac9bd1b030a81a53068e1b Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 14:43:05 +0300 Subject: [PATCH 02/26] Don't use get_latest_server_stats Use get_server_stats_by_id instead of get_latest_server_stats The reasoning behind this is that all usages of get_latest_server_stats had `DatabaseShortcuts.return_rows(latest)[0]`, which get_server_stats_by_id already did --- app/classes/controllers/server_perms_controller.py | 4 ++-- app/classes/controllers/servers_controller.py | 8 ++++---- app/classes/web/routes/api/servers/server/stats.py | 5 +---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/app/classes/controllers/server_perms_controller.py b/app/classes/controllers/server_perms_controller.py index 95395fec..4c9dded7 100644 --- a/app/classes/controllers/server_perms_controller.py +++ b/app/classes/controllers/server_perms_controller.py @@ -107,11 +107,11 @@ class ServerPermsController: ) for server in authorized_servers: - latest = HelperServerStats.get_latest_server_stats(server.get("server_id")) + latest = HelperServerStats.get_server_stats_by_id(server.get("server_id")) server_data.append( { "server_data": server, - "stats": DatabaseShortcuts.return_rows(latest)[0], + "stats": latest, } ) return server_data diff --git a/app/classes/controllers/servers_controller.py b/app/classes/controllers/servers_controller.py index f9aa2143..edeccc44 100644 --- a/app/classes/controllers/servers_controller.py +++ b/app/classes/controllers/servers_controller.py @@ -153,7 +153,7 @@ class ServersController: ) for server in authorized_servers: - latest = HelperServerStats.get_latest_server_stats(server.get("server_id")) + latest = HelperServerStats.get_server_stats_by_id(server.get("server_id")) key_permissions = PermissionsServers.get_api_key_permissions_list( api_key, server.get("server_id") ) @@ -164,7 +164,7 @@ class ServersController: server_data.append( { "server_data": server, - "stats": DatabaseShortcuts.return_rows(latest)[0], + "stats": latest, "user_command_permission": user_command_permission, } ) @@ -176,7 +176,7 @@ class ServersController: authorized_servers = ServersController.get_authorized_servers(user_id) for server in authorized_servers: - latest = HelperServerStats.get_latest_server_stats(server.get("server_id")) + latest = HelperServerStats.get_server_stats_by_id(server.get("server_id")) # TODO user_permissions = PermissionsServers.get_user_id_permissions_list( user_id, server.get("server_id") @@ -188,7 +188,7 @@ class ServersController: server_data.append( { "server_data": server, - "stats": DatabaseShortcuts.return_rows(latest)[0], + "stats": latest, "user_command_permission": user_command_permission, } ) diff --git a/app/classes/web/routes/api/servers/server/stats.py b/app/classes/web/routes/api/servers/server/stats.py index 2a8bc23e..f1bf4c42 100644 --- a/app/classes/web/routes/api/servers/server/stats.py +++ b/app/classes/web/routes/api/servers/server/stats.py @@ -1,5 +1,4 @@ import logging -from playhouse.shortcuts import model_to_dict from app.classes.models.server_stats import HelperServerStats from app.classes.web.base_api_handler import BaseApiHandler @@ -21,8 +20,6 @@ class ApiServersServerStatsHandler(BaseApiHandler): 200, { "status": "ok", - "data": model_to_dict( - HelperServerStats.get_latest_server_stats(server_id)[0] - ), + "data": HelperServerStats.get_server_stats_by_id(server_id), }, ) From a937a94ac83986568766a14949cadf23f5546ee8 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 14:49:17 +0300 Subject: [PATCH 03/26] Use a better variable name for stat disappear date * Rename occurences of `last_week` with `minimum_to_exist` --- app/classes/minecraft/stats.py | 4 ++-- app/classes/models/server_stats.py | 4 ++-- app/classes/shared/server.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index 95b781ef..13247cbc 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -258,6 +258,6 @@ class Stats: # delete old data max_age = self.helper.get_setting("history_max_age") now = datetime.datetime.now() - last_week = now.day - max_age + minimum_to_exist = now.day - max_age - HostStats.delete().where(HostStats.time < last_week).execute() + HostStats.delete().where(HostStats.time < minimum_to_exist).execute() diff --git a/app/classes/models/server_stats.py b/app/classes/models/server_stats.py index 7e16f444..f2576aa4 100644 --- a/app/classes/models/server_stats.py +++ b/app/classes/models/server_stats.py @@ -176,9 +176,9 @@ class HelperServerStats: ).execute() @staticmethod - def remove_old_stats(server_id, last_week): + def remove_old_stats(server_id, minimum_to_exist): HelperServerStats.select_database(server_id) - ServerStats.delete().where(ServerStats.created < last_week).execute() + ServerStats.delete().where(ServerStats.created < minimum_to_exist).execute() @staticmethod def get_latest_server_stats(server_id): diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 7d85ad59..abbfc975 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1450,6 +1450,6 @@ class Server: # delete old data max_age = self.helper.get_setting("history_max_age") now = datetime.datetime.now() - last_week = now.day - max_age + minimum_to_exist = now.day - max_age - HelperServerStats.remove_old_stats(server.get("id", 0), last_week) + HelperServerStats.remove_old_stats(server.get("id", 0), minimum_to_exist) From 83018c68602c0de16bc538cbd12cf4d4aae5b7b5 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 16:14:42 +0300 Subject: [PATCH 04/26] Calculate the min stats date with timedelta This is more correct than just using days. I've also tested this working in a completely separate Peewee project. --- app/classes/minecraft/stats.py | 2 +- app/classes/shared/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index 13247cbc..5a36da8c 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -258,6 +258,6 @@ class Stats: # delete old data max_age = self.helper.get_setting("history_max_age") now = datetime.datetime.now() - minimum_to_exist = now.day - max_age + minimum_to_exist = now - datetime.timedelta(days=max_age) HostStats.delete().where(HostStats.time < minimum_to_exist).execute() diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index abbfc975..89327a85 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1450,6 +1450,6 @@ class Server: # delete old data max_age = self.helper.get_setting("history_max_age") now = datetime.datetime.now() - minimum_to_exist = now.day - max_age + minimum_to_exist = now - datetime.timedelta(days=max_age) HelperServerStats.remove_old_stats(server.get("id", 0), minimum_to_exist) From 4e84eee5a567235cc2454c76230350311e0a7275 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 16:15:51 +0300 Subject: [PATCH 05/26] Rename server stats variables to server_stats --- app/classes/models/server_stats.py | 36 ++++++++++++++++-------------- app/classes/shared/server.py | 6 ++--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/app/classes/models/server_stats.py b/app/classes/models/server_stats.py index f2576aa4..e526f247 100644 --- a/app/classes/models/server_stats.py +++ b/app/classes/models/server_stats.py @@ -147,8 +147,8 @@ class HelperServerStats: return server_data @staticmethod - def insert_server_stats(server): - server_id = server.get("id", 0) + def insert_server_stats(server_stats): + server_id = server_stats.get("id", 0) HelperServerStats.select_database(server_id) if server_id == 0: @@ -157,21 +157,23 @@ class HelperServerStats: ServerStats.insert( { - ServerStats.server_id: server.get("id", 0), - ServerStats.started: server.get("started", ""), - ServerStats.running: server.get("running", False), - ServerStats.cpu: server.get("cpu", 0), - ServerStats.mem: server.get("mem", 0), - ServerStats.mem_percent: server.get("mem_percent", 0), - ServerStats.world_name: server.get("world_name", ""), - ServerStats.world_size: server.get("world_size", ""), - ServerStats.server_port: server.get("server_port", ""), - ServerStats.int_ping_results: server.get("int_ping_results", False), - ServerStats.online: server.get("online", False), - ServerStats.max: server.get("max", False), - ServerStats.players: server.get("players", False), - ServerStats.desc: server.get("desc", False), - ServerStats.version: server.get("version", False), + ServerStats.server_id: server_stats.get("id", 0), + ServerStats.started: server_stats.get("started", ""), + ServerStats.running: server_stats.get("running", False), + ServerStats.cpu: server_stats.get("cpu", 0), + ServerStats.mem: server_stats.get("mem", 0), + ServerStats.mem_percent: server_stats.get("mem_percent", 0), + ServerStats.world_name: server_stats.get("world_name", ""), + ServerStats.world_size: server_stats.get("world_size", ""), + ServerStats.server_port: server_stats.get("server_port", ""), + ServerStats.int_ping_results: server_stats.get( + "int_ping_results", False + ), + ServerStats.online: server_stats.get("online", False), + ServerStats.max: server_stats.get("max", False), + ServerStats.players: server_stats.get("players", False), + ServerStats.desc: server_stats.get("desc", False), + ServerStats.version: server_stats.get("version", False), } ).execute() diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 89327a85..2700e805 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1444,12 +1444,12 @@ class Server: def record_server_stats(self): - server = self.get_servers_stats() - HelperServerStats.insert_server_stats(server) + server_stats = self.get_servers_stats() + HelperServerStats.insert_server_stats(server_stats) # delete old data max_age = self.helper.get_setting("history_max_age") now = datetime.datetime.now() minimum_to_exist = now - datetime.timedelta(days=max_age) - HelperServerStats.remove_old_stats(server.get("id", 0), minimum_to_exist) + HelperServerStats.remove_old_stats(server_stats.get("id", 0), minimum_to_exist) From 0eac0721ea58f2bc9ca4afe31afe83f8eb13a9d2 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 16:17:01 +0300 Subject: [PATCH 06/26] Remove unused DatabaseShortcuts imports --- app/classes/controllers/server_perms_controller.py | 1 - app/classes/controllers/servers_controller.py | 1 - 2 files changed, 2 deletions(-) diff --git a/app/classes/controllers/server_perms_controller.py b/app/classes/controllers/server_perms_controller.py index 4c9dded7..78b2c53d 100644 --- a/app/classes/controllers/server_perms_controller.py +++ b/app/classes/controllers/server_perms_controller.py @@ -8,7 +8,6 @@ from app.classes.models.users import HelperUsers, ApiKeys from app.classes.models.roles import HelperRoles from app.classes.models.servers import HelperServers from app.classes.models.server_stats import HelperServerStats -from app.classes.shared.main_models import DatabaseShortcuts logger = logging.getLogger(__name__) diff --git a/app/classes/controllers/servers_controller.py b/app/classes/controllers/servers_controller.py index edeccc44..03f4cd02 100644 --- a/app/classes/controllers/servers_controller.py +++ b/app/classes/controllers/servers_controller.py @@ -12,7 +12,6 @@ from app.classes.models.server_permissions import ( EnumPermissionsServer, ) from app.classes.shared.helpers import Helpers -from app.classes.shared.main_models import DatabaseShortcuts logger = logging.getLogger(__name__) From 8e13b4e11dc17f90939999d2887b09315df24e40 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 16:21:34 +0300 Subject: [PATCH 07/26] Simplify helpers setting methods --- app/classes/shared/helpers.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 778bfbfb..81606ba3 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -194,7 +194,6 @@ class Helpers: return cmd_out def get_setting(self, key, default_return=False): - try: with open(self.settings_file, "r", encoding="utf-8") as f: data = json.load(f) @@ -202,10 +201,8 @@ class Helpers: if key in data.keys(): return data.get(key) - else: - logger.error(f"Config File Error: setting {key} does not exist") - Console.error(f"Config File Error: setting {key} does not exist") - return default_return + logger.error(f'Config File Error: Setting "{key}" does not exist') + Console.error(f'Config File Error: Setting "{key}" does not exist') except Exception as e: logger.critical( @@ -217,22 +214,19 @@ class Helpers: return default_return - def set_setting(self, key, new_value, default_return=False): - + def set_setting(self, key, new_value): try: with open(self.settings_file, "r", encoding="utf-8") as f: data = json.load(f) if key in data.keys(): data[key] = new_value + with open(self.settings_file, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + return True - else: - logger.error(f"Config File Error: setting {key} does not exist") - Console.error(f"Config File Error: setting {key} does not exist") - return default_return - - with open(self.settings_file, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) + logger.error(f'Config File Error: Setting "{key}" does not exist') + Console.error(f'Config File Error: Setting "{key}" does not exist') except Exception as e: logger.critical( @@ -241,6 +235,7 @@ class Helpers: Console.critical( f"Config File Error: Unable to read {self.settings_file} due to {e}" ) + return False @staticmethod def get_local_ip(): From ca450f21ed73cfeedd4f8117329406b7e1acc997 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 16:25:49 +0300 Subject: [PATCH 08/26] Document set_passwd and threads prompt commands Help messages: set_passwd: Set a user's password. Example: set_passwd admin threads: Get all of the Python threads used by Crafty --- app/classes/shared/command.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/classes/shared/command.py b/app/classes/shared/command.py index 2fb222cb..160a6b59 100644 --- a/app/classes/shared/command.py +++ b/app/classes/shared/command.py @@ -119,3 +119,9 @@ class MainPrompt(cmd.Cmd): def help_import3(self): Console.help("Import users and servers from Crafty 3") + + def help_set_passwd(self): + Console.help("Set a user's password. Example: set_passwd admin") + + def help_threads(self): + Console.help("Get all of the Python threads used by Crafty") From 3956d9c6991d3159643b9bf0f4f12dc8ba6fe77f Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 17:44:09 +0300 Subject: [PATCH 09/26] Remove stdin sending from command_watcher Reasons: * It's unused in our program * It can be a start to security vulnerabilities. I've seen many routes add commands to `Commands` without validation. --- app/classes/shared/tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index 58e1cc2d..83d124d2 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -121,8 +121,6 @@ class TasksManager: elif command == "update_executable": svr.jar_update() - else: - svr.send_command(command) HelpersManagement.mark_command_complete(cmd.command_id) time.sleep(1) From 2e51fa9629f454e0a9d0890165bd5788f388747f Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 18:41:16 +0300 Subject: [PATCH 10/26] Remove newline characters from API file logs --- app/classes/web/routes/api/servers/server/logs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/classes/web/routes/api/servers/server/logs.py b/app/classes/web/routes/api/servers/server/logs.py index efb18630..e06410e0 100644 --- a/app/classes/web/routes/api/servers/server/logs.py +++ b/app/classes/web/routes/api/servers/server/logs.py @@ -45,6 +45,9 @@ class ApiServersServerLogsHandler(BaseApiHandler): self.helper.get_os_understandable_path(server_data["log_path"]), log_lines, ) + + # Remove newline characters from the end of the lines + raw_lines = [line.rstrip("\r\n") for line in raw_lines] else: raw_lines = ServerOutBuf.lines.get(server_id, []) From 8ce7a96071a26aef4871ec42ed25c47a2536b353 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 25 May 2022 18:41:41 +0300 Subject: [PATCH 11/26] Use a better ANSI escape regex in the API logs --- app/classes/web/routes/api/servers/server/logs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/classes/web/routes/api/servers/server/logs.py b/app/classes/web/routes/api/servers/server/logs.py index e06410e0..b29333f6 100644 --- a/app/classes/web/routes/api/servers/server/logs.py +++ b/app/classes/web/routes/api/servers/server/logs.py @@ -8,6 +8,8 @@ from app.classes.web.base_api_handler import BaseApiHandler logger = logging.getLogger(__name__) +ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + class ApiServersServerLogsHandler(BaseApiHandler): def get(self, server_id: str): @@ -56,9 +58,7 @@ class ApiServersServerLogsHandler(BaseApiHandler): for line in raw_lines: try: if not disable_ansi_strip: - line = re.sub( - "(\033\\[(0;)?[0-9]*[A-z]?(;[0-9])?m?)|(> )", "", line - ) + line = ansi_escape.sub("", line) line = re.sub("[A-z]{2}\b\b", "", line) line = html.escape(line) From 47fc398f267a35537bb206996d286e5ba8a05271 Mon Sep 17 00:00:00 2001 From: luukas Date: Thu, 26 May 2022 00:31:58 +0300 Subject: [PATCH 12/26] Fix CORS preflight --- app/classes/web/base_api_handler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/classes/web/base_api_handler.py b/app/classes/web/base_api_handler.py index 24d7328d..9fa028b4 100644 --- a/app/classes/web/base_api_handler.py +++ b/app/classes/web/base_api_handler.py @@ -19,5 +19,12 @@ class BaseApiHandler(BaseHandler): delete = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]] patch = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]] put = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]] - options = _unimplemented_method # type: Callable[..., Optional[Awaitable[None]]] # }}} + + def options(self, *_, **__): + """ + Fix CORS + """ + # no body + self.set_status(204) + self.finish() From d1beb2f125d9611f6422b4cade909f50cfc7f5dc Mon Sep 17 00:00:00 2001 From: luukas Date: Sat, 28 May 2022 19:00:25 +0300 Subject: [PATCH 13/26] Redirect stderr to null during psutil's import. On some systems /proc might be unavailable and psutil would freak out printing several exceptions to stderr but strangely not raising the exceptions for upper scope to handle --- app/classes/minecraft/bedrock_ping.py | 7 ++++++- app/classes/minecraft/stats.py | 6 +++++- app/classes/shared/helpers.py | 7 +++++-- app/classes/shared/null_writer.py | 10 ++++++++++ app/classes/shared/server.py | 8 ++++++-- 5 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 app/classes/shared/null_writer.py diff --git a/app/classes/minecraft/bedrock_ping.py b/app/classes/minecraft/bedrock_ping.py index 556a221a..d2be6449 100644 --- a/app/classes/minecraft/bedrock_ping.py +++ b/app/classes/minecraft/bedrock_ping.py @@ -1,7 +1,12 @@ +from contextlib import redirect_stderr import os import socket import time -import psutil + +from app.classes.shared.null_writer import NullWriter + +with redirect_stderr(NullWriter()): + import psutil class BedrockPing: diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index 5a36da8c..0f4208a8 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -1,16 +1,20 @@ from __future__ import annotations +from contextlib import redirect_stderr import json import logging import datetime import base64 import typing as t -import psutil +from app.classes.shared.null_writer import NullWriter from app.classes.minecraft.mc_ping import ping from app.classes.models.management import HostStats from app.classes.models.servers import HelperServers from app.classes.shared.helpers import Helpers +with redirect_stderr(NullWriter()): + import psutil + if t.TYPE_CHECKING: from app.classes.shared.main_controller import Controller diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 81606ba3..4ac928b5 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -16,15 +16,18 @@ import pathlib import ctypes from datetime import datetime from socket import gethostname -from contextlib import suppress -import psutil +from contextlib import redirect_stderr, suppress +from app.classes.shared.null_writer import NullWriter from app.classes.shared.console import Console from app.classes.shared.installer import installer from app.classes.shared.file_helpers import FileHelpers from app.classes.shared.translation import Translation from app.classes.web.websocket_helper import WebSocketHelper +with redirect_stderr(NullWriter()): + import psutil + logger = logging.getLogger(__name__) try: diff --git a/app/classes/shared/null_writer.py b/app/classes/shared/null_writer.py new file mode 100644 index 00000000..55a09460 --- /dev/null +++ b/app/classes/shared/null_writer.py @@ -0,0 +1,10 @@ +import logging +import os + +logger = logging.getLogger(__name__) + + +class NullWriter: + def write(self, data): + if os.environ["CRAFTY_LOG_NULLWRITER"] == "true": + logger.debug(data) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 2700e805..4e2c104f 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1,3 +1,4 @@ +from contextlib import redirect_stderr import os import re import time @@ -8,8 +9,6 @@ import logging.config import subprocess import html import tempfile -import psutil -from psutil import NoSuchProcess # TZLocal is set as a hidden import on win pipeline from tzlocal import get_localzone @@ -25,6 +24,11 @@ from app.classes.models.server_permissions import PermissionsServers from app.classes.shared.console import Console from app.classes.shared.helpers import Helpers from app.classes.shared.file_helpers import FileHelpers +from app.classes.shared.null_writer import NullWriter + +with redirect_stderr(NullWriter()): + import psutil + from psutil import NoSuchProcess logger = logging.getLogger(__name__) From 0df5fedf2ba6a322bc0779e0f312fda2c2822d1c Mon Sep 17 00:00:00 2001 From: luukas Date: Sat, 28 May 2022 19:32:54 +0300 Subject: [PATCH 14/26] Add try-excepts for host stats --- app/classes/minecraft/stats.py | 80 ++++++++++++++++++++++++++++------ app/classes/shared/server.py | 4 +- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index 0f4208a8..f500af28 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -56,30 +56,66 @@ class Stats: helper: Helpers controller: Controller + @staticmethod + def try_get_boot_time(): + try: + return datetime.datetime.fromtimestamp( + psutil.boot_time(), datetime.timezone.utc + ) + except Exception as e: + logger.debug("error while getting boot time", exc=e) + # unix epoch with no timezone data + return datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + + @staticmethod + def try_get_cpu_usage(): + try: + return psutil.cpu_percent(interval=0.5) / psutil.cpu_count() + except Exception as e: + logger.debug("error while getting cpu percentage", exc=e) + return -1 + def __init__(self, helper, controller): self.helper = helper self.controller = controller def get_node_stats(self) -> NodeStatsReturnDict: - boot_time = datetime.datetime.fromtimestamp(psutil.boot_time()) try: cpu_freq = psutil.cpu_freq() except NotImplementedError: cpu_freq = psutil._common.scpufreq(current=0, min=0, max=0) memory = psutil.virtual_memory() - node_stats: NodeStatsDict = { - "boot_time": str(boot_time), - "cpu_usage": psutil.cpu_percent(interval=0.5) / psutil.cpu_count(), - "cpu_count": psutil.cpu_count(), - "cpu_cur_freq": round(cpu_freq[0], 2), - "cpu_max_freq": cpu_freq[2], - "mem_percent": memory.percent, - "mem_usage_raw": memory.used, - "mem_usage": Helpers.human_readable_file_size(memory.used), - "mem_total_raw": memory.total, - "mem_total": Helpers.human_readable_file_size(memory.total), - "disk_data": self._all_disk_usage(), - } + try: + node_stats: NodeStatsDict = { + "boot_time": str(Stats.try_get_boot_time()), + "cpu_usage": Stats.try_get_cpu_usage(), + "cpu_count": psutil.cpu_count(), + "cpu_cur_freq": round(cpu_freq[0], 2), + "cpu_max_freq": cpu_freq[2], + "mem_percent": memory.percent, + "mem_usage_raw": memory.used, + "mem_usage": Helpers.human_readable_file_size(memory.used), + "mem_total_raw": memory.total, + "mem_total": Helpers.human_readable_file_size(memory.total), + "disk_data": Stats._try_all_disk_usage(), + } + except Exception as e: + logger.debug("error while getting host stats", exc=e) + node_stats: NodeStatsDict = { + "boot_time": str( + datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + ), + "cpu_usage": -1, + "cpu_count": -1, + "cpu_cur_freq": -1, + "cpu_max_freq": -1, + "mem_percent": -1, + "mem_usage_raw": -1, + "mem_usage": "", + "mem_total_raw": -1, + "mem_total": "", + "disk_data": [], + } # server_stats = self.get_servers_stats() # data['servers'] = server_stats @@ -87,6 +123,14 @@ class Stats: "node_stats": node_stats, } + @staticmethod + def _try_get_process_stats(process): + try: + return Stats._get_process_stats(process) + except Exception as e: + logger.debug("error while getting process stats", exc=e) + return {"cpu_usage": -1, "memory_usage": -1, "mem_percentage": -1} + @staticmethod def _get_process_stats(process): if process is None: @@ -126,6 +170,14 @@ class Stats: } return process_stats + @staticmethod + def _try_all_disk_usage(): + try: + return Stats._all_disk_usage() + except Exception as e: + logger.debug("error while getting disk data", exc=e) + return [] + # Source: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py @staticmethod def _all_disk_usage() -> t.List[DiskDataDict]: diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 4e2c104f..4e5db4c3 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1201,7 +1201,7 @@ class Server: server_path = server["path"] # process stats - p_stats = Stats._get_process_stats(self.process) + p_stats = Stats._try_get_process_stats(self.process) # TODO: search server properties file for possible override of 127.0.0.1 internal_ip = server["server_ip"] @@ -1334,7 +1334,7 @@ class Server: server_path = server_dt["path"] # process stats - p_stats = Stats._get_process_stats(self.process) + p_stats = Stats._try_get_process_stats(self.process) # TODO: search server properties file for possible override of 127.0.0.1 # internal_ip = server['server_ip'] From b0e356f9995922b36a05cac0ebb982457fd3d193 Mon Sep 17 00:00:00 2001 From: luukas Date: Sat, 28 May 2022 19:43:37 +0300 Subject: [PATCH 15/26] Fix host stats error messages --- app/classes/minecraft/stats.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index f500af28..28564945 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -63,7 +63,7 @@ class Stats: psutil.boot_time(), datetime.timezone.utc ) except Exception as e: - logger.debug("error while getting boot time", exc=e) + logger.debug(f"error while getting boot time due to {e}") # unix epoch with no timezone data return datetime.datetime.fromtimestamp(0, datetime.timezone.utc) @@ -72,7 +72,7 @@ class Stats: try: return psutil.cpu_percent(interval=0.5) / psutil.cpu_count() except Exception as e: - logger.debug("error while getting cpu percentage", exc=e) + logger.debug(f"error while getting cpu percentage due to {e}") return -1 def __init__(self, helper, controller): @@ -100,7 +100,7 @@ class Stats: "disk_data": Stats._try_all_disk_usage(), } except Exception as e: - logger.debug("error while getting host stats", exc=e) + logger.debug(f"error while getting host stats due to {e}") node_stats: NodeStatsDict = { "boot_time": str( datetime.datetime.fromtimestamp(0, datetime.timezone.utc) @@ -128,7 +128,7 @@ class Stats: try: return Stats._get_process_stats(process) except Exception as e: - logger.debug("error while getting process stats", exc=e) + logger.debug(f"error while getting process stats due to {e}") return {"cpu_usage": -1, "memory_usage": -1, "mem_percentage": -1} @staticmethod @@ -175,7 +175,7 @@ class Stats: try: return Stats._all_disk_usage() except Exception as e: - logger.debug("error while getting disk data", exc=e) + logger.debug(f"error while getting disk data due to {e}") return [] # Source: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py From 7885b2c8f70bb7245bc5f947880264441cf0e528 Mon Sep 17 00:00:00 2001 From: luukas Date: Sat, 28 May 2022 21:11:09 +0300 Subject: [PATCH 16/26] Fix null writer --- app/classes/shared/null_writer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/classes/shared/null_writer.py b/app/classes/shared/null_writer.py index 55a09460..cced3b7e 100644 --- a/app/classes/shared/null_writer.py +++ b/app/classes/shared/null_writer.py @@ -6,5 +6,7 @@ logger = logging.getLogger(__name__) class NullWriter: def write(self, data): - if os.environ["CRAFTY_LOG_NULLWRITER"] == "true": + if os.environ.get("CRAFTY_LOG_NULLWRITER", "false") == "true": logger.debug(data) + if os.environ.get("CRAFTY_PRINT_NULLWRITER", "false") == "true": + print(data) From 290c398198e666493534cc33ce456171d39c8d58 Mon Sep 17 00:00:00 2001 From: luukas Date: Mon, 30 May 2022 18:23:37 +0300 Subject: [PATCH 17/26] Add command queue stdin commands back --- app/classes/shared/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index 83d124d2..898ef9c3 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -121,6 +121,9 @@ class TasksManager: elif command == "update_executable": svr.jar_update() + else: + svr.send_command(command) + HelpersManagement.mark_command_complete(cmd.command_id) time.sleep(1) From e0b0e52bd514883baaec56df814897d548703ee6 Mon Sep 17 00:00:00 2001 From: luukas Date: Tue, 31 May 2022 00:11:17 +0300 Subject: [PATCH 18/26] Add stdin route for the API --- app/classes/shared/server.py | 1 + app/classes/web/routes/api/api_handlers.py | 6 +++ .../web/routes/api/servers/server/stdin.py | 48 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 app/classes/web/routes/api/servers/server/stdin.py diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 4e5db4c3..78afa666 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -630,6 +630,7 @@ class Server: # send it self.process.stdin.write(f"{command}\n".encode("utf-8")) self.process.stdin.flush() + return True def crash_detected(self, name): diff --git a/app/classes/web/routes/api/api_handlers.py b/app/classes/web/routes/api/api_handlers.py index 6c032811..e5f72b48 100644 --- a/app/classes/web/routes/api/api_handlers.py +++ b/app/classes/web/routes/api/api_handlers.py @@ -22,6 +22,7 @@ from app.classes.web.routes.api.servers.server.public import ( ApiServersServerPublicHandler, ) from app.classes.web.routes.api.servers.server.stats import ApiServersServerStatsHandler +from app.classes.web.routes.api.servers.server.stdin import ApiServersServerStdinHandler from app.classes.web.routes.api.servers.server.users import ApiServersServerUsersHandler from app.classes.web.routes.api.users.index import ApiUsersIndexHandler from app.classes.web.routes.api.users.user.index import ApiUsersUserIndexHandler @@ -127,6 +128,11 @@ def api_handlers(handler_args): ApiServersServerPublicHandler, handler_args, ), + ( + r"/api/v2/servers/([0-9]+)/stdin/?", + ApiServersServerStdinHandler, + handler_args, + ), ( r"/api/v2/roles/?", ApiRolesIndexHandler, diff --git a/app/classes/web/routes/api/servers/server/stdin.py b/app/classes/web/routes/api/servers/server/stdin.py new file mode 100644 index 00000000..aff8697d --- /dev/null +++ b/app/classes/web/routes/api/servers/server/stdin.py @@ -0,0 +1,48 @@ +import logging + +from peewee import DoesNotExist +from app.classes.models.server_permissions import EnumPermissionsServer +from app.classes.web.base_api_handler import BaseApiHandler + + +logger = logging.getLogger(__name__) + + +class ApiServersServerStdinHandler(BaseApiHandler): + def post(self, server_id: str): + auth_data = self.authenticate_user() + if not auth_data: + return + + if server_id not in [str(x["server_id"]) for x in auth_data[0]]: + # if the user doesn't have access to the server, return an error + return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"}) + + if ( + EnumPermissionsServer.COMMANDS + not in self.controller.server_perms.get_user_id_permissions_list( + auth_data[4]["user_id"], server_id + ) + ): + # if the user doesn't have Commands permission, return an error + return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"}) + + try: + svr = self.controller.get_server_obj(server_id) + except DoesNotExist: + logger.critical( + "Something has gone VERY wrong! " + "Crafty can't access the server object. " + "Please report this to the devs" + ) + return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"}) + + if svr.send_command(self.request.body.decode("utf-8")): + return self.finish_json( + 200, + {"status": "ok"}, + ) + self.finish_json( + 200, + {"status": "error", "error": "SERVER_NOT_RUNNING"}, + ) From d486fdd4c88992f01a7387efaf6dde2b85ee48dd Mon Sep 17 00:00:00 2001 From: DarthLeo1000YT Date: Mon, 30 May 2022 20:57:19 -0400 Subject: [PATCH 19/26] add jquery back to the main base.html template. --- app/frontend/templates/base.html | 5 +++++ 1 file changed, 5 insertions(+) mode change 100644 => 100755 app/frontend/templates/base.html diff --git a/app/frontend/templates/base.html b/app/frontend/templates/base.html old mode 100644 new mode 100755 index 0311241d..5f51533a --- a/app/frontend/templates/base.html +++ b/app/frontend/templates/base.html @@ -21,6 +21,11 @@ + + + + + From 45a1b835fbe5fa5431f59b31857237a8c850e0ff Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 1 Jun 2022 14:17:33 +0300 Subject: [PATCH 20/26] Fix server error checking in API stdin endpoint --- app/classes/web/routes/api/servers/server/stdin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/classes/web/routes/api/servers/server/stdin.py b/app/classes/web/routes/api/servers/server/stdin.py index aff8697d..a52f0c0d 100644 --- a/app/classes/web/routes/api/servers/server/stdin.py +++ b/app/classes/web/routes/api/servers/server/stdin.py @@ -1,6 +1,5 @@ import logging -from peewee import DoesNotExist from app.classes.models.server_permissions import EnumPermissionsServer from app.classes.web.base_api_handler import BaseApiHandler @@ -27,9 +26,9 @@ class ApiServersServerStdinHandler(BaseApiHandler): # if the user doesn't have Commands permission, return an error return self.finish_json(400, {"status": "error", "error": "NOT_AUTHORIZED"}) - try: - svr = self.controller.get_server_obj(server_id) - except DoesNotExist: + svr = self.controller.get_server_obj_optional(server_id) + if svr is None: + # It's in auth_data[0] but not as a Server object logger.critical( "Something has gone VERY wrong! " "Crafty can't access the server object. " From 256c6567fdfb6368ea1a023d8304d090187e98e5 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Wed, 1 Jun 2022 18:28:24 +0100 Subject: [PATCH 21/26] Remove temporary directory on backup completion When running backups, the temporary directory wasn't being properly removed. Added logic to remove the temporary directory upon backup completion, cleaned up the try/except a bit. --- app/classes/shared/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 96ce4935..8510d481 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -877,7 +877,6 @@ class Server: os.remove(Helpers.get_os_understandable_path(oldfile_path)) self.is_backingup = False - FileHelpers.del_dirs(temp_dir) logger.info(f"Backup of server: {self.name} completed") self.server_scheduler.remove_job("backup_" + str(self.server_id)) results = {"percent": 100, "total_files": 0, "current_file": 0} @@ -900,7 +899,6 @@ class Server: ).format(self.name), ) time.sleep(3) - return except: logger.exception( f"Failed to create backup of server {self.name} (ID {self.server_id})" @@ -915,7 +913,10 @@ class Server: results, ) self.is_backingup = False - return + finally: + print(temp_dir) + FileHelpers.del_dirs(temp_dir) + return def backup_status(self, source_path, dest_path): results = Helpers.calc_percent(source_path, dest_path) From de96844276b202f2165d5ea7e2b933987d1d66c7 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Wed, 1 Jun 2022 18:34:04 +0100 Subject: [PATCH 22/26] Remove useless return from `a_backup_server` (Pylint R1711) --- app/classes/shared/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 8510d481..af92c9df 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -916,7 +916,6 @@ class Server: finally: print(temp_dir) FileHelpers.del_dirs(temp_dir) - return def backup_status(self, source_path, dest_path): results = Helpers.calc_percent(source_path, dest_path) From fdc88451b97c935fe61deab97414d8d423483068 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 1 Jun 2022 15:11:37 -0400 Subject: [PATCH 23/26] Add exception for permissions in helpers check --- app/classes/shared/helpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 778bfbfb..d9d82c34 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -690,6 +690,9 @@ class Helpers: # directory already exists - non-blocking error except FileExistsError: pass + except PermissionError as e: + logger.critical(f"Check generated exception due to permssion error: {e}") + pass def create_self_signed_cert(self, cert_dir=None): From 3946a926bfc446111b3c56d4697dcf41a7307084 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 1 Jun 2022 15:17:13 -0400 Subject: [PATCH 24/26] Remove print statement --- app/classes/shared/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index af92c9df..6044f81f 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -914,7 +914,6 @@ class Server: ) self.is_backingup = False finally: - print(temp_dir) FileHelpers.del_dirs(temp_dir) def backup_status(self, source_path, dest_path): From 28ffbe4627b28b9240fd5e705f8caa844e844ded Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 1 Jun 2022 15:20:16 -0400 Subject: [PATCH 25/26] Remove unnecessary pass --- app/classes/shared/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index d9d82c34..857c4978 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -692,7 +692,6 @@ class Helpers: pass except PermissionError as e: logger.critical(f"Check generated exception due to permssion error: {e}") - pass def create_self_signed_cert(self, cert_dir=None): From 6474663a645e3c370b2254460e0a6ed064bb28b2 Mon Sep 17 00:00:00 2001 From: luukas Date: Wed, 1 Jun 2022 23:25:53 +0300 Subject: [PATCH 26/26] Fix Finnish translations --- app/translations/fi_FI.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/app/translations/fi_FI.json b/app/translations/fi_FI.json index 2d38e4b7..66f0be89 100644 --- a/app/translations/fi_FI.json +++ b/app/translations/fi_FI.json @@ -65,7 +65,7 @@ "cannotSee": "Etkö näe kaikkea?", "cannotSeeOnMobile": "Etkö näe kaikkea mobiililaitteella?", "cannotSeeOnMobile2": "Yritä vierittää taulukkoa sivuttain.", - "clone": "Klooni", + "clone": "Kloonaa", "cpuCores": "Suorittimen ytimet", "cpuCurFreq": "Nykyinen kellotaajuus", "cpuMaxFreq": "Maksimi kellotaajuus", @@ -91,10 +91,10 @@ "server": "Palvelin", "servers": "Palvelimet", "size": "Palvelimen hakemiston koko", - "start": "Alkaa", + "start": "Käynnistä", "starting": "Myöhästynyt lähtö", "status": "Tila", - "stop": "Lopettaa", + "stop": "Sammuta", "version": "Versio", "welcome": "Tervetuloa Crafty Controller" }, @@ -126,12 +126,12 @@ "print": "Tulosta" }, "decimal": "", - "emptyTable": "Ei tietoja saatavilla taulukossa", - "info": "Näytetään _START_-_END_ / _TOTAL_ merkinnästä", - "infoEmpty": "Showing 0 to 0 of 0 entries", - "infoFiltered": "(filtered from _MAX_ total entries)", + "emptyTable": "Tietoja ei löytynyt", + "info": "Näytetään rivit _START_ - _END_ (yhteensä _TOTAL_ )", + "infoEmpty": "Näytetään 0 - 0 (yhteensä 0)", + "infoFiltered": "(suodatettu _MAX_ tuloksen joukosta)", "infoPostFix": "", - "lengthMenu": "Show _MENU_ entries", + "lengthMenu": "Näytä kerralla _MENU_ riviä", "loadingRecords": "Ladataan...", "paginate": { "first": "Ensimmäinen", @@ -411,8 +411,8 @@ "children": "Linkitetyt lapsitehtävät: ", "command": "Komento", "command-explain": "Minkä komennon haluat meidän suorittavan? Älä sisällytä '/'", - "cron": "Cron", - "cron-explain": "Kirjoita Cron-merkkijonosi", + "cron": "CRON", + "cron-explain": "Kirjoita CRON-ilmaisusi", "custom": "Mukautettu komento", "days": "Päivää", "enabled": "Käytössä", @@ -421,15 +421,15 @@ "interval-explain": "Kuinka usein haluat tämän ajoituksen suoritettavan?", "minutes": "Minuuttia", "offset": "Viivepoikkeama", - "offset-explain": "Kuinka kauan meidän pitäisi odottaa tämän suorittamista ensimmäisen tehtävän suorittamisen jälkeen? (sekunteissa)", + "offset-explain": "Kuinka kauan Craftyn pitäisi odottaa tämän suorittamista ensimmäisen tehtävän suorittamisen jälkeen? (sekunteissa)", "one-time": "Poista suorituksen jälkeen", "parent": "Valitse isäntäajoitus", "parent-explain": "Minkä ajoituksen pitäisi käynnistää tämä?", - "reaction": "Reaktio", + "reaction": "Ketjureaktio", "restart": "Uudelleenkäynnistä palvelin", "start": "Käynnistä palvelin", "stop": "Sammuta palvelin", - "time": "Aika", + "time": "Päivänaika", "time-explain": "Mihin aikaan haluat ajoituksen suoritettavan?" }, "serverSchedules": {