From 54d8ee4b7d0e3e7d00ce432e2900f9e33430b95e Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 10:32:26 -0600 Subject: [PATCH 01/18] Add migration for excluded backup directories Changes the column 'directories' in the backups table to 'excluded_dirs' so that we're only storing a list of excluded directories --- app/migrations/20220227162446_backup_options.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 app/migrations/20220227162446_backup_options.py diff --git a/app/migrations/20220227162446_backup_options.py b/app/migrations/20220227162446_backup_options.py new file mode 100644 index 00000000..1239dd54 --- /dev/null +++ b/app/migrations/20220227162446_backup_options.py @@ -0,0 +1,8 @@ +# Generated by database migrator +import peewee + +def migrate(migrator, db): + migrator.rename_column('backups', 'directories', 'excluded_dirs') + +def rollback(migrator, db): + migrator.rename_column('backups', 'excluded_dirs', 'directories') \ No newline at end of file From 29800eee3c4ba67b3db8e6c0270c9ac5a481a6ef Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 12:35:48 -0600 Subject: [PATCH 02/18] Add methods for excluded backup directories Adds get, add, and delete methods for excluded backup directories --- app/classes/models/management.py | 36 +++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/app/classes/models/management.py b/app/classes/models/management.py index b652ff11..b089df70 100644 --- a/app/classes/models/management.py +++ b/app/classes/models/management.py @@ -125,7 +125,7 @@ class Schedules(Model): # Backups Class #************************************************************************************************ class Backups(Model): - directories = CharField(null=True) + excluded_dirs = CharField(null=True) max_backups = IntegerField() server_id = ForeignKeyField(Servers, backref='backups_server') class Meta: @@ -311,28 +311,28 @@ class helpers_management: row = Backups.select().where(Backups.server_id == server_id).join(Servers)[0] conf = { "backup_path": row.server_id.backup_path, - "directories": row.directories, + "excluded_dirs": row.excluded_dirs, "max_backups": row.max_backups, "server_id": row.server_id.server_id } except IndexError: conf = { "backup_path": None, - "directories": None, + "excluded_dirs": None, "max_backups": 0, "server_id": server_id } return conf @staticmethod - def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None): + def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None): logger.debug(f"Updating server {server_id} backup config with {locals()}") if Backups.select().where(Backups.server_id == server_id).count() != 0: new_row = False conf = {} else: conf = { - "directories": None, + "excluded_dirs": None, "max_backups": 0, "server_id": server_id } @@ -354,6 +354,32 @@ class helpers_management: Servers.update(backup_path=backup_path).where(Servers.server_id == server_id) Backups.create(**conf) logger.debug("Creating new backup record.") + + @staticmethod + def get_excluded_backup_dirs(server_id: int): + excluded_dirs = Backups.select(Backups.excluded_dirs).where(Backups.server_id == server_id).execute() + dir_list = excluded_dirs.split(",") + return dir_list + + @staticmethod + def add_excluded_backup_dir(server_id: int, dir_to_add: str): + dir_list = management_helper.get_excluded_backup_dirs() + if dir_to_add not in dir_list: + dir_list.append(dir_to_add) + excluded_dirs = ",".join(dir_list) + management_helper.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs) + else: + logger.debug(f"Not adding {dir_to_add} to excluded directories - already in the excluded directory list for server ID {server_id}") + + @staticmethod + def del_excluded_backup_dir(server_id: int, dir_to_del: str): + dir_list = management_helper.get_excluded_backup_dirs() + if dir_to_del in dir_list: + dir_list.remove(dir_to_del) + excluded_dirs = ",".join(dir_list) + management_helper.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs) + else: + logger.debug(f"Not removing {dir_to_del} from excluded directories - not in the excluded directory list for server ID {server_id}") management_helper = helpers_management() From 7c5bb0984cfb3e409b0ef19ed84a914144aa41f0 Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 13:11:22 -0600 Subject: [PATCH 03/18] Add logic for removing excluded files/dirs from backups Updated backup thread to copy all server files to a temporary directory, iterate through excluded directory list, remove excluded directories/files, then create the backup archive --- app/classes/shared/server.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 82e1c1da..0a01ff90 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -8,6 +8,7 @@ import logging.config import shutil import subprocess import html +import tempfile from apscheduler.schedulers.background import BackgroundScheduler #TZLocal is set as a hidden import on win pipeline from tzlocal import get_localzone @@ -568,14 +569,30 @@ class Server: backup_filename = f"{self.settings['backup_path']}/{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}" logger.info(f"Creating backup of server '{self.settings['server_name']}'" + f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'") - shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', self.server_path) + + # shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', self.server_path) + tempDir = tempfile.mkdtemp() + shutil.copytree(self.server_path, tempDir) + excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id) + server_dir = helper.get_os_understandable_path(self.settings['path']) + for dir in excluded_dirs: + temp_dir = helper.get_os_understandable_path(dir).replace(server_dir, helper.get_os_understandable_path(tempDir)) + if os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + else: + os.remove(temp_dir) + + shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', tempDir) + while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0: backup_list = self.list_backups() oldfile = backup_list[0] oldfile_path = f"{conf['backup_path']}/{oldfile['path']}" logger.info(f"Removing old backup '{oldfile['path']}'") os.remove(helper.get_os_understandable_path(oldfile_path)) + self.is_backingup = False + shutil.rmtree(tempDir) logger.info(f"Backup of server: {self.name} completed") return except: From eaca8acf72bce838ea79285949313ab1e0b42ac2 Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 13:17:14 -0600 Subject: [PATCH 04/18] Clean up backup server method for readability --- app/classes/shared/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 0a01ff90..71e1148a 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -570,11 +570,11 @@ class Server: logger.info(f"Creating backup of server '{self.settings['server_name']}'" + f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'") - # shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', self.server_path) tempDir = tempfile.mkdtemp() shutil.copytree(self.server_path, tempDir) excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id) server_dir = helper.get_os_understandable_path(self.settings['path']) + for dir in excluded_dirs: temp_dir = helper.get_os_understandable_path(dir).replace(server_dir, helper.get_os_understandable_path(tempDir)) if os.path.isdir(temp_dir): From cf5bfe88c172b028ce086138d86ed64a70195a7a Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 13:21:54 -0600 Subject: [PATCH 05/18] Better documentation for backup logic --- app/classes/shared/server.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 71e1148a..dec37d8a 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -574,13 +574,18 @@ class Server: shutil.copytree(self.server_path, tempDir) excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id) server_dir = helper.get_os_understandable_path(self.settings['path']) - + for dir in excluded_dirs: - temp_dir = helper.get_os_understandable_path(dir).replace(server_dir, helper.get_os_understandable_path(tempDir)) - if os.path.isdir(temp_dir): - shutil.rmtree(temp_dir) + # Take the full path of the excluded dir and replace the server path with the temp path + # This is so that we're only deleting excluded dirs from the temp path and not the server path + excluded_dir = helper.get_os_understandable_path(dir).replace(server_dir, helper.get_os_understandable_path(tempDir)) + # Next, check to see if it is a directory + if os.path.isdir(excluded_dir): + # If it is a directory, recursively delete the entire directory from the backup + shutil.rmtree(excluded_dir) else: - os.remove(temp_dir) + # If not, just remove the file + os.remove(excluded_dir) shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', tempDir) From 4b484782c61866872bf6d6d350f3f8a400c08106 Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 13:38:16 -0600 Subject: [PATCH 06/18] Fix excluded directories not being separatable Excluded directories were being pulled from the database as not a string, so I just used the get_backup_config() method to pull exclusions as a string so I could split it --- app/classes/models/management.py | 2 +- app/classes/shared/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/classes/models/management.py b/app/classes/models/management.py index b089df70..887b2d27 100644 --- a/app/classes/models/management.py +++ b/app/classes/models/management.py @@ -357,7 +357,7 @@ class helpers_management: @staticmethod def get_excluded_backup_dirs(server_id: int): - excluded_dirs = Backups.select(Backups.excluded_dirs).where(Backups.server_id == server_id).execute() + excluded_dirs = helpers_management.get_backup_config(server_id)['excluded_dirs'] dir_list = excluded_dirs.split(",") return dir_list diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index dec37d8a..06d6b453 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -571,7 +571,7 @@ class Server: f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'") tempDir = tempfile.mkdtemp() - shutil.copytree(self.server_path, tempDir) + shutil.copytree(self.server_path, tempDir, dirs_exist_ok=True) excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id) server_dir = helper.get_os_understandable_path(self.settings['path']) From 6d7ddf8494663d07fbb4b2af27434f5e8c0f4cc5 Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Sun, 27 Feb 2022 15:08:03 -0600 Subject: [PATCH 07/18] Add controllers for backup exclusion logic --- app/classes/controllers/management_controller.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/classes/controllers/management_controller.py b/app/classes/controllers/management_controller.py index d256c52a..1d1fdc8f 100644 --- a/app/classes/controllers/management_controller.py +++ b/app/classes/controllers/management_controller.py @@ -106,3 +106,15 @@ class Management_Controller: @staticmethod def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None): return management_helper.set_backup_config(server_id, backup_path, max_backups) + + @staticmethod + def get_excluded_backup_dirs(server_id: int): + return management_helper.get_excluded_backup_dirs(server_id) + + @staticmethod + def add_excluded_backup_dir(server_id: int, dir_to_add: str): + management_helper.add_excluded_backup_dir(server_id, dir_to_add) + + @staticmethod + def del_excluded_backup_dir(server_id: int, dir_to_del: str): + management_helper.del_excluded_backup_dir(server_id, dir_to_del) \ No newline at end of file From e7d78cadd405f633975c0065fa41d941bda20e8b Mon Sep 17 00:00:00 2001 From: Andrew Date: Sun, 27 Feb 2022 16:15:40 -0500 Subject: [PATCH 08/18] Initial frontend for backup omissions --- app/classes/shared/helpers.py | 57 +++++ app/classes/web/ajax_handler.py | 21 ++ .../templates/panel/server_backup.html | 207 ++++++++++++++++++ .../templates/server/bedrock_wizard.html | 20 +- app/frontend/templates/server/wizard.html | 20 +- 5 files changed, 313 insertions(+), 12 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index a4566df5..72a583a7 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -737,6 +737,57 @@ class Helpers: return output + @staticmethod + def generate_backup_tree(folder, output=""): + file_list = os.listdir(folder) + file_list = sorted(file_list, key=str.casefold) + output += \ + f"""