From b766ae10e9713d8e978f19b05c3190e632412d3a Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Sat, 25 Feb 2023 15:40:38 -0500 Subject: [PATCH 01/13] Only copy bedrock_server executable on update --- app/classes/shared/file_helpers.py | 71 ++++++++++++++++++------------ app/classes/shared/server.py | 5 ++- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/app/classes/shared/file_helpers.py b/app/classes/shared/file_helpers.py index e26220bb..f7052943 100644 --- a/app/classes/shared/file_helpers.py +++ b/app/classes/shared/file_helpers.py @@ -283,39 +283,56 @@ class FileHelpers: return True @staticmethod - def unzip_file(zip_path): - new_dir_list = zip_path.split("/") - new_dir = "" - for i in range(len(new_dir_list) - 1): - if i == 0: - new_dir += new_dir_list[i] - else: - new_dir += "/" + new_dir_list[i] - + def unzip_file(zip_path, single_item=False): + # Get directory without zipfile name + new_dir = pathlib.Path(zip_path).parents[0] + # make sure we're able to access the zip file if Helpers.check_file_perms(zip_path) and os.path.isfile(zip_path): + # make sure the directory we're unzipping this to exists Helpers.ensure_dir_exists(new_dir) + # we'll make a temporary directory to unzip this to. temp_dir = tempfile.mkdtemp() try: with zipfile.ZipFile(zip_path, "r") as zip_ref: + # we'll extract this to the temp dir using zipfile module zip_ref.extractall(temp_dir) - full_root_path = temp_dir - for item in os.listdir(full_root_path): - if os.path.isdir(os.path.join(full_root_path, item)): - try: - FileHelpers.move_dir_exist( - os.path.join(full_root_path, item), - os.path.join(new_dir, item), - ) - except Exception as ex: - logger.error(f"ERROR IN ZIP IMPORT: {ex}") - else: - try: - FileHelpers.move_file( - os.path.join(full_root_path, item), - os.path.join(new_dir, item), - ) - except Exception as ex: - logger.error(f"ERROR IN ZIP IMPORT: {ex}") + + # we'll check if the single item parameter was passed + # if it was the developer only wants one file back + # probably for a server executable update. + if not single_item: + # we'll iterate through the top level directory moving everything + # out of the temp directory and into it's final home. + for item in os.listdir(temp_dir): + # we handle files and dirs differently or we'll crash out. + if os.path.isdir(os.path.join(temp_dir, item)): + try: + FileHelpers.move_dir_exist( + os.path.join(temp_dir, item), + os.path.join(new_dir, item), + ) + except Exception as ex: + logger.error(f"ERROR IN ZIP IMPORT: {ex}") + else: + try: + FileHelpers.move_file( + os.path.join(temp_dir, item), + os.path.join(new_dir, item), + ) + except Exception as ex: + logger.error(f"ERROR IN ZIP IMPORT: {ex}") + else: + # if there is a single item the correct file name should be passed + # we'll just try to move that one file based on the name provided + # then we'll move along. + try: + FileHelpers.move_file( + os.path.join(temp_dir, single_item), + os.path.join(new_dir, single_item), + ) + except FileNotFoundError: + logger.error("Could not unpack single file in bedrock update.") + return "false" except Exception as ex: Console.error(ex) else: diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 713d31bc..7b1958da 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1320,7 +1320,10 @@ class ServerInstance: unzip_path = os.path.join(self.settings["path"], "bedrock_server.zip") unzip_path = self.helper.wtol_path(unzip_path) # unzips archive that was downloaded. - FileHelpers.unzip_file(unzip_path) + if self.helper.is_os_windows(): + FileHelpers.unzip_file(unzip_path, "bedrock_server.exe") + else: + FileHelpers.unzip_file(unzip_path, "bedrock_server") # adjusts permissions for execution if os is not windows if not self.helper.is_os_windows(): os.chmod( From 1dc731c9f7f267000d249ae36405b8c4e1e3b9b7 Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Sat, 25 Feb 2023 16:15:50 -0500 Subject: [PATCH 02/13] Rework logic --- app/classes/shared/file_helpers.py | 62 ++++++++++++------------------ app/classes/shared/server.py | 6 +-- 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/app/classes/shared/file_helpers.py b/app/classes/shared/file_helpers.py index f7052943..57dcc5bf 100644 --- a/app/classes/shared/file_helpers.py +++ b/app/classes/shared/file_helpers.py @@ -283,7 +283,8 @@ class FileHelpers: return True @staticmethod - def unzip_file(zip_path, single_item=False): + def unzip_file(zip_path, server_update=False): + ignored_names = ["server.properties", "permissions.json", "allowlist.json"] # Get directory without zipfile name new_dir = pathlib.Path(zip_path).parents[0] # make sure we're able to access the zip file @@ -296,43 +297,30 @@ class FileHelpers: with zipfile.ZipFile(zip_path, "r") as zip_ref: # we'll extract this to the temp dir using zipfile module zip_ref.extractall(temp_dir) + # we'll iterate through the top level directory moving everything + # out of the temp directory and into it's final home. + for item in os.listdir(temp_dir): + # if the file is one of our ignored names we'll skip it + if item in ignored_names and server_update: + continue + # we handle files and dirs differently or we'll crash out. + if os.path.isdir(os.path.join(temp_dir, item)): + try: + FileHelpers.move_dir_exist( + os.path.join(temp_dir, item), + os.path.join(new_dir, item), + ) + except Exception as ex: + logger.error(f"ERROR IN ZIP IMPORT: {ex}") + else: - # we'll check if the single item parameter was passed - # if it was the developer only wants one file back - # probably for a server executable update. - if not single_item: - # we'll iterate through the top level directory moving everything - # out of the temp directory and into it's final home. - for item in os.listdir(temp_dir): - # we handle files and dirs differently or we'll crash out. - if os.path.isdir(os.path.join(temp_dir, item)): - try: - FileHelpers.move_dir_exist( - os.path.join(temp_dir, item), - os.path.join(new_dir, item), - ) - except Exception as ex: - logger.error(f"ERROR IN ZIP IMPORT: {ex}") - else: - try: - FileHelpers.move_file( - os.path.join(temp_dir, item), - os.path.join(new_dir, item), - ) - except Exception as ex: - logger.error(f"ERROR IN ZIP IMPORT: {ex}") - else: - # if there is a single item the correct file name should be passed - # we'll just try to move that one file based on the name provided - # then we'll move along. - try: - FileHelpers.move_file( - os.path.join(temp_dir, single_item), - os.path.join(new_dir, single_item), - ) - except FileNotFoundError: - logger.error("Could not unpack single file in bedrock update.") - return "false" + try: + FileHelpers.move_file( + os.path.join(temp_dir, item), + os.path.join(new_dir, item), + ) + except Exception as ex: + logger.error(f"ERROR IN ZIP IMPORT: {ex}") except Exception as ex: Console.error(ex) else: diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 7b1958da..9ca54a90 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -1320,10 +1320,7 @@ class ServerInstance: unzip_path = os.path.join(self.settings["path"], "bedrock_server.zip") unzip_path = self.helper.wtol_path(unzip_path) # unzips archive that was downloaded. - if self.helper.is_os_windows(): - FileHelpers.unzip_file(unzip_path, "bedrock_server.exe") - else: - FileHelpers.unzip_file(unzip_path, "bedrock_server") + FileHelpers.unzip_file(unzip_path, server_update=True) # adjusts permissions for execution if os is not windows if not self.helper.is_os_windows(): os.chmod( @@ -1337,6 +1334,7 @@ class ServerInstance: logger.critical( f"Failed to download bedrock executable for update \n{e}" ) + downloaded = False if downloaded: logger.info("Executable updated successfully. Starting Server") From a50fa25ae510249b5a3f8d0fc1cfd9a74e0127c0 Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Sat, 25 Feb 2023 16:25:09 -0500 Subject: [PATCH 03/13] Appease black --- app/classes/shared/file_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/classes/shared/file_helpers.py b/app/classes/shared/file_helpers.py index 57dcc5bf..4005e965 100644 --- a/app/classes/shared/file_helpers.py +++ b/app/classes/shared/file_helpers.py @@ -313,7 +313,6 @@ class FileHelpers: except Exception as ex: logger.error(f"ERROR IN ZIP IMPORT: {ex}") else: - try: FileHelpers.move_file( os.path.join(temp_dir, item), From 6e35d4ae972af3b7a447cf7737efc693bb56489e Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Sun, 26 Feb 2023 16:38:27 -0500 Subject: [PATCH 04/13] Default delete crafty logs after 2 weeks --- app/classes/shared/helpers.py | 1 + app/classes/shared/tasks.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 4374c99f..96b3fac8 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -441,6 +441,7 @@ class Helpers: "reset_secrets_on_next_boot": False, "monitored_mounts": mounts, "dir_size_poll_freq_minutes": 5, + "crafty_logs_delete_after_days": 14, } def get_all_settings(self): diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index 3a57cf90..9c1f52b0 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -751,10 +751,39 @@ class TasksManager: logger.debug("Could not clear out file from import directory") def log_watcher(self): - self.controller.servers.check_for_old_logs() + self.check_for_old_logs() self.scheduler.add_job( - self.controller.servers.check_for_old_logs, + self.check_for_old_logs, "interval", hours=6, id="log-mgmt", ) + + def check_for_old_logs(self): + # check for server logs first + self.controller.servers.check_for_old_logs() + # check for crafty logs now + logs_path = os.path.join(self.controller.project_root, "logs") + logs_delete_after = int( + self.helper.get_setting("crafty_logs_delete_after_days") + ) + latest_log_files = [ + "session.log", + "schedule.log", + "tornado-access.log", + "session.log", + "commander.log", + ] + + log_files = list( + filter( + lambda val: val not in latest_log_files, + os.listdir(logs_path), + ) + ) + for log_file in log_files: + log_file_path = os.path.join(logs_path, log_file) + if Helpers.check_file_exists( + log_file_path + ) and Helpers.is_file_older_than_x_days(log_file_path, logs_delete_after): + os.remove(log_file_path) From efb9c9844f69d8dc91eab81d792745288aa2b3be Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Wed, 1 Mar 2023 13:35:36 -0500 Subject: [PATCH 05/13] Fix bug where failed server could not delete --- app/classes/web/panel_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 27758bef..7c3640ac 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -539,6 +539,7 @@ class PanelHandler(BaseHandler): "auto_start": server_temp_obj["auto_start"], "crash_detection": server_temp_obj["crash_detection"], "show_status": server_temp_obj["show_status"], + "ignored_exits": server_temp_obj["ignored_exits"], }, "running": False, "crashed": False, From 566463cd5525e5ffcecbc173b6bee556798b459d Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Wed, 1 Mar 2023 14:22:33 -0500 Subject: [PATCH 06/13] Fix bug where "servers" was not appended --- app/classes/shared/main_controller.py | 18 ++++++++++-------- app/classes/web/panel_handler.py | 6 +++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/classes/shared/main_controller.py b/app/classes/shared/main_controller.py index 8bcf0bf5..836e5dc9 100644 --- a/app/classes/shared/main_controller.py +++ b/app/classes/shared/main_controller.py @@ -1023,7 +1023,9 @@ class Controller: move_thread.start() def t_update_master_server_dir(self, new_server_path, user_id): + print("in update") new_server_path = self.helper.wtol_path(new_server_path) + new_server_path = os.path.join(new_server_path, "servers") self.helper.websocket_helper.broadcast_page( "/panel/panel_config", "move_status", "Checking dir" ) @@ -1055,7 +1057,7 @@ class Controller: self.helper.websocket_helper.broadcast_page( "/panel/panel_config", "move_status", "Checking permissions" ) - if not self.helper.ensure_dir_exists(os.path.join(new_server_path, "servers")): + if not self.helper.ensure_dir_exists(new_server_path): self.helper.websocket_helper.broadcast_user( user_id, "send_start_error", @@ -1074,8 +1076,8 @@ class Controller: # move the servers for server in servers: server_path = server.get("path") - new_server_path = os.path.join( - new_server_path, "servers", server.get("server_uuid") + new_local_server_path = os.path.join( + new_server_path, server.get("server_uuid") ) if os.path.isdir(server_path): self.helper.websocket_helper.broadcast_page( @@ -1086,7 +1088,7 @@ class Controller: try: self.file_helper.move_dir( server_path, - new_server_path, + new_local_server_path, ) except FileExistsError as e: logger.error(f"Failed to move server with error: {e}") @@ -1096,19 +1098,19 @@ class Controller: # reset executable path if current_master in server["executable"]: server_obj.executable = str(server["executable"]).replace( - current_master, new_server_path + current_master, new_local_server_path ) # reset run command path if current_master in server["execution_command"]: server_obj.execution_command = str(server["execution_command"]).replace( - current_master, new_server_path + current_master, new_local_server_path ) # reset log path if current_master in server["log_path"]: server_obj.log_path = str(server["log_path"]).replace( - current_master, new_server_path + current_master, new_local_server_path ) - server_obj.path = new_server_path + server_obj.path = new_local_server_path failed = False for s in self.servers.failed_servers: if int(s["server_id"]) == int(server.get("server_id")): diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 27758bef..11fa328d 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -848,9 +848,9 @@ class PanelHandler(BaseHandler): page_data["auth-servers"] = auth_servers page_data["role-servers"] = auth_role_servers page_data["user-roles"] = user_roles - page_data[ - "servers_dir" - ] = self.controller.management.get_master_server_dir() + page_data["servers_dir"], _tail = os.path.split( + self.controller.management.get_master_server_dir() + ) page_data["users"] = self.controller.users.user_query(exec_user["user_id"]) page_data["roles"] = self.controller.users.user_role_query( From bd0c975009af1f9f09d088f45ca15c6866595322 Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Wed, 1 Mar 2023 14:30:59 -0500 Subject: [PATCH 07/13] Remove print statement --- app/classes/shared/main_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/classes/shared/main_controller.py b/app/classes/shared/main_controller.py index 836e5dc9..95872884 100644 --- a/app/classes/shared/main_controller.py +++ b/app/classes/shared/main_controller.py @@ -1023,7 +1023,6 @@ class Controller: move_thread.start() def t_update_master_server_dir(self, new_server_path, user_id): - print("in update") new_server_path = self.helper.wtol_path(new_server_path) new_server_path = os.path.join(new_server_path, "servers") self.helper.websocket_helper.broadcast_page( From 39190ed43be06d2e495d6c47c374664661b66b31 Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Thu, 2 Mar 2023 13:40:45 -0500 Subject: [PATCH 08/13] Default to not delete logs --- app/classes/shared/helpers.py | 2 +- app/classes/shared/tasks.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 96b3fac8..cf108bcd 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -441,7 +441,7 @@ class Helpers: "reset_secrets_on_next_boot": False, "monitored_mounts": mounts, "dir_size_poll_freq_minutes": 5, - "crafty_logs_delete_after_days": 14, + "crafty_logs_delete_after_days": 0, } def get_all_settings(self): diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index 9c1f52b0..8f51842b 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -774,16 +774,19 @@ class TasksManager: "session.log", "commander.log", ] - - log_files = list( - filter( - lambda val: val not in latest_log_files, - os.listdir(logs_path), + # we won't delete if delete logs after is set to 0 + if logs_delete_after != 0: + log_files = list( + filter( + lambda val: val not in latest_log_files, + os.listdir(logs_path), + ) ) - ) - for log_file in log_files: - log_file_path = os.path.join(logs_path, log_file) - if Helpers.check_file_exists( - log_file_path - ) and Helpers.is_file_older_than_x_days(log_file_path, logs_delete_after): - os.remove(log_file_path) + for log_file in log_files: + log_file_path = os.path.join(logs_path, log_file) + if Helpers.check_file_exists( + log_file_path + ) and Helpers.is_file_older_than_x_days( + log_file_path, logs_delete_after + ): + os.remove(log_file_path) From 57202553423830d5948920cc541409b4d617d954 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Sat, 4 Mar 2023 13:51:13 +0000 Subject: [PATCH 09/13] Update changelog !562 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5ea41d..c1f91f6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Bump Cryptography/pyOpenSSL for CVE-2023-23931 ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/554)) - Fix debug logging to only display with the -v (verbose) flag ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/560)) - Optimize world size calculation ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/550)) +- Only copy bedrock_server executable on update ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/562)) ### Tweaks - Cleanup authentication helpers ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/545)) - Optimize file upload progress WS ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/546)) From 05f6bffb00d6902363d444b7891e8104accee23c Mon Sep 17 00:00:00 2001 From: Zedifus Date: Sat, 4 Mar 2023 14:12:50 +0000 Subject: [PATCH 10/13] Update changelog !563 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5ea41d..1b44901a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Truncate sidebar servers to a max of 10 ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/552)) - Upgrade to FA 6. Add Translations ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/549))([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/558)) - Forge installer and Java Detection improvements ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/559)) +- Crafty log clean up -config option ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/563)) ### Lang - Add additional translations to backups page strings ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/543)) - Add additional missing translations ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/549)) From 8e9554e4c8dac686b0f320064fc17505b1e67736 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Sat, 4 Mar 2023 15:09:17 +0000 Subject: [PATCH 11/13] Update changelog !566 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97723300..c1d90a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Fix debug logging to only display with the -v (verbose) flag ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/560)) - Optimize world size calculation ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/550)) - Only copy bedrock_server executable on update ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/562)) +- Fix bug where unloaded servers could not be deleted ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/566)) ### Tweaks - Cleanup authentication helpers ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/545)) - Optimize file upload progress WS ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/546)) From b164bc0ab0fe10253deca351b188e7e7c6596a74 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Sat, 4 Mar 2023 15:21:21 +0000 Subject: [PATCH 12/13] Update changelog !567 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1d90a11..f3cdcf9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Optimize world size calculation ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/550)) - Only copy bedrock_server executable on update ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/562)) - Fix bug where unloaded servers could not be deleted ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/566)) +- Fix bug where "servers" was not appended ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/567)) ### Tweaks - Cleanup authentication helpers ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/545)) - Optimize file upload progress WS ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/546)) From 77d80584d1d2bf7b0b4b56a9ba6e386f9a48e492 Mon Sep 17 00:00:00 2001 From: Zedifus Date: Sat, 4 Mar 2023 15:35:12 +0000 Subject: [PATCH 13/13] Add DHub to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 618a4b89..5d5435e6 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Discord Server - https://discord.gg/9VJPhCE Git Repository - https://gitlab.com/crafty-controller/crafty-4 +Docker Hub - [arcadiatechnology/crafty-4](https://hub.docker.com/r/arcadiatechnology/crafty-4) +
## Basic Docker Usage 🐳