From 826d4d081f4afc9e41f8b16dbcbeeeff5c4586c6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 2 Mar 2022 09:48:03 -0500 Subject: [PATCH 1/5] Improve session.lock checking. --- app/classes/shared/helpers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index d9877f90..c9777346 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -17,6 +17,7 @@ import ctypes from datetime import datetime from socket import gethostname from contextlib import suppress +import psutil from requests import get from app.classes.web.websocket_helper import websocket_helper @@ -481,11 +482,18 @@ class Helpers: pid = data.get('pid') started = data.get('started') console.critical(f"Another Crafty Controller agent seems to be running...\npid: {pid} \nstarted on: {started}") + if psutil.pid_exists(pid): + logger.critical("Found running crafty process. Exiting.") + sys.exit(1) + else: + logger.info("No process found for pid. Assuming crafty crashed. Deleting stale session.lock") + os.remove(self.session_file) + except Exception as e: logger.error(f"Failed to locate existing session.lock with error: {e} ") console.error(f"Failed to locate existing session.lock with error: {e} ") - sys.exit(1) + sys.exit(1) pid = os.getpid() now = datetime.now() From 8e874ca997d252852d34259cbcacb2f9560ba7e1 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 2 Mar 2022 10:30:40 -0500 Subject: [PATCH 2/5] Add option to disable language files through config.json --- app/classes/web/panel_handler.py | 12 +++++++----- app/config/config.json | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index df2bb932..3e4a753a 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -606,10 +606,11 @@ class PanelHandler(BaseHandler): page_data['super-disabled'] = '' else: page_data['super-disabled'] = 'disabled' - for file in os.listdir(os.path.join(helper.root_dir, 'app', 'translations')): + for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))): if file.endswith('.json'): - if file != str(page_data['languages'][0] + '.json'): - page_data['languages'].append(file.split('.')[0]) + if file not in helper.get_setting('disabled_language_files'): + if file != str(page_data['languages'][0] + '.json'): + page_data['languages'].append(file.split('.')[0]) template = "panel/panel_edit_user.html" @@ -738,8 +739,9 @@ class PanelHandler(BaseHandler): for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))): if file.endswith('.json'): - if file != str(page_data['languages'][0] + '.json'): - page_data['languages'].append(file.split('.')[0]) + if file not in helper.get_setting('disabled_language_files'): + if file != str(page_data['languages'][0] + '.json'): + page_data['languages'].append(file.split('.')[0]) if user_id is None: self.redirect("/panel/error?error=Invalid User ID") diff --git a/app/config/config.json b/app/config/config.json index 7f3ff5f6..aad33f03 100644 --- a/app/config/config.json +++ b/app/config/config.json @@ -14,6 +14,7 @@ "virtual_terminal_lines": 70, "max_log_lines": 700, "max_audit_entries": 300, + "disabled_language_files": ["lol_EN.json", ""], "keywords": ["help", "chunk"], "allow_nsfw_profile_pictures": false } From 7707940743516969c61dc60bf42c26f6652d615c Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 2 Mar 2022 10:36:24 -0500 Subject: [PATCH 3/5] Appease the linter --- app/classes/minecraft/stats.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index ee2d8acc..b12dd180 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -1,4 +1,3 @@ -import os import json import logging import datetime @@ -114,7 +113,7 @@ class Stats: def get_world_size(server_path): total_size = 0 - + total_size = helper.get_dir_size(server_path) level_total_size = helper.human_readable_file_size(total_size) From 15610210b9e8b9cb6769f55ade8600184c7fad6b Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 2 Mar 2022 10:49:03 -0500 Subject: [PATCH 4/5] Fix bug in schedule creation where if interval was days an execption would be thrown for a missing key --- app/classes/web/panel_handler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 3e4a753a..76243e09 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -1191,7 +1191,9 @@ class PanelHandler(BaseHandler): "start_time": sch_time, "enabled": enabled, "one_time": one_time, - "cron_string": '' + "cron_string": '', + "parent": None, + "delay": 0 } elif difficulty == "reaction": job_data = { @@ -1339,7 +1341,9 @@ class PanelHandler(BaseHandler): "start_time": sch_time, "enabled": enabled, "one_time": one_time, - "cron_string": '' + "cron_string": '', + "parent": None, + "delay": 0 } elif difficulty == "advanced": job_data = { From e0b8fb35fe768a38d32eac8a7dfbc7e52a8c474b Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 2 Mar 2022 11:34:26 -0500 Subject: [PATCH 5/5] Add option to compress backups --- .../controllers/management_controller.py | 4 ++-- app/classes/models/management.py | 13 +++++++---- app/classes/shared/file_helpers.py | 23 ++++++++++++++++++- app/classes/shared/server.py | 7 +++++- app/classes/web/panel_handler.py | 3 ++- .../templates/panel/server_backup.html | 10 ++++++++ app/migrations/20220302_compress_backups.py | 16 +++++++++++++ app/translations/en_EN.json | 3 ++- 8 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 app/migrations/20220302_compress_backups.py diff --git a/app/classes/controllers/management_controller.py b/app/classes/controllers/management_controller.py index 6efe492a..59419ca0 100644 --- a/app/classes/controllers/management_controller.py +++ b/app/classes/controllers/management_controller.py @@ -104,8 +104,8 @@ class Management_Controller: return management_helper.get_backup_config(server_id) @staticmethod - def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None): - return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs) + def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False,): + return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs, compress) @staticmethod def get_excluded_backup_dirs(server_id: int): diff --git a/app/classes/models/management.py b/app/classes/models/management.py index e49dc4f7..8cd6e527 100644 --- a/app/classes/models/management.py +++ b/app/classes/models/management.py @@ -128,6 +128,7 @@ class Backups(Model): excluded_dirs = CharField(null=True) max_backups = IntegerField() server_id = ForeignKeyField(Servers, backref='backups_server') + compress = BooleanField(default=False) class Meta: table_name = 'backups' database = database @@ -313,19 +314,21 @@ class helpers_management: "backup_path": row.server_id.backup_path, "excluded_dirs": row.excluded_dirs, "max_backups": row.max_backups, - "server_id": row.server_id.server_id + "server_id": row.server_id.server_id, + "compress": row.compress } except IndexError: conf = { "backup_path": None, "excluded_dirs": None, "max_backups": 0, - "server_id": server_id + "server_id": server_id, + "compress": False, } return conf @staticmethod - def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None): + def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False): logger.debug(f"Updating server {server_id} backup config with {locals()}") if Backups.select().where(Backups.server_id == server_id).count() != 0: new_row = False @@ -334,7 +337,8 @@ class helpers_management: conf = { "excluded_dirs": None, "max_backups": 0, - "server_id": server_id + "server_id": server_id, + "compress": False } new_row = True if max_backups is not None: @@ -342,6 +346,7 @@ class helpers_management: if excluded_dirs is not None: dirs_to_exclude = ",".join(excluded_dirs) conf['excluded_dirs'] = dirs_to_exclude + conf['compress'] = compress if not new_row: with database.atomic(): if backup_path is not None: diff --git a/app/classes/shared/file_helpers.py b/app/classes/shared/file_helpers.py index d1b72ee6..39d23b04 100644 --- a/app/classes/shared/file_helpers.py +++ b/app/classes/shared/file_helpers.py @@ -9,7 +9,7 @@ from app.classes.shared.console import console logger = logging.getLogger(__name__) try: - from zipfile import ZipFile + from zipfile import ZipFile, ZIP_DEFLATED except ModuleNotFoundError as err: logger.critical(f"Import Error: Unable to load {err.name} module", exc_info=True) @@ -87,4 +87,25 @@ class FileHelpers: return True + @staticmethod + def make_compressed_archive(path_to_destination, path_to_zip): + # create a ZipFile object + path_to_destination += '.zip' + with ZipFile(path_to_destination, 'w', ZIP_DEFLATED) as z: + for root, _dirs, files in os.walk(path_to_zip, topdown=True): + ziproot = path_to_zip + for file in files: + try: + logger.info(f"backing up: {os.path.join(root, file)}") + if os.name == "nt": + z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, ""), file)) + else: + z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, "/"), file)) + + except Exception as e: + logger.warning(f"Error backing up: {os.path.join(root, file)}! - Error was: {e}") + + + return True + file_helper = FileHelpers() diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 107815be..7b799fd4 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -604,7 +604,12 @@ class Server: else: # If not, just remove the file os.remove(excluded_dir) - file_helper.make_archive(helper.get_os_understandable_path(backup_filename), tempDir) + if conf['compress']: + logger.debug("Found compress backup to be true. Calling compressed archive") + file_helper.make_compressed_archive(helper.get_os_understandable_path(backup_filename), tempDir) + else: + logger.debug("Found compress backup to be false. Calling NON-compressed archive") + file_helper.make_archive(helper.get_os_understandable_path(backup_filename), tempDir) while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0: backup_list = self.list_backups() diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 76243e09..11469e33 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -1061,6 +1061,7 @@ class PanelHandler(BaseHandler): logger.debug(self.request.arguments) server_id = self.get_argument('id', None) server_obj = self.controller.servers.get_server_obj(server_id) + compress = self.get_argument('compress', False) check_changed = self.get_argument('changed') if str(check_changed) == str(1): checked = self.get_body_arguments('root_path') @@ -1091,7 +1092,7 @@ class PanelHandler(BaseHandler): server_obj = self.controller.servers.get_server_obj(server_id) server_obj.backup_path = backup_path self.controller.servers.update_server(server_obj) - self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked) + self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked, compress=bool(compress)) self.controller.management.add_to_audit_log(exec_user['user_id'], f"Edited server {server_id}: updated backups", diff --git a/app/frontend/templates/panel/server_backup.html b/app/frontend/templates/panel/server_backup.html index 29f09fe7..e2498a05 100644 --- a/app/frontend/templates/panel/server_backup.html +++ b/app/frontend/templates/panel/server_backup.html @@ -55,6 +55,16 @@
+
+
+ + {% if data['backup_config']['compress'] %} + {{ translate('serverBackups', 'compress', data['lang']) }} + {% else %} + {{ translate('serverBackups', 'compress', data['lang']) }} + {% end %}
diff --git a/app/migrations/20220302_compress_backups.py b/app/migrations/20220302_compress_backups.py new file mode 100644 index 00000000..35f16f73 --- /dev/null +++ b/app/migrations/20220302_compress_backups.py @@ -0,0 +1,16 @@ +# Generated by database migrator +import peewee + +def migrate(migrator, database, **kwargs): + migrator.add_columns('backups', compress=peewee.BooleanField(default=False)) + """ + Write your migrations here. + """ + + + +def rollback(migrator, database, **kwargs): + migrator.drop_columns('backups', ['compress']) + """ + Write your rollback migrations here. + """ diff --git a/app/translations/en_EN.json b/app/translations/en_EN.json index 314361e4..f585b127 100644 --- a/app/translations/en_EN.json +++ b/app/translations/en_EN.json @@ -243,7 +243,8 @@ "excludedBackups": "Excluded Paths: ", "excludedChoose": "Choose the paths you wish to exclude from your backups", "clickExclude": "Click to select Exclusions", - "exclusionsTitle": "Backup Exclusions" + "exclusionsTitle": "Backup Exclusions", + "compress": "Compress Backup" }, "serverFiles": { "noscript": "The file manager does not work without JavaScript",