From 5c85a575ee9874427b57a4666b30f9d57caa95cd Mon Sep 17 00:00:00 2001 From: Scott R <86810816+xithical@users.noreply.github.com> Date: Fri, 13 Aug 2021 19:45:15 -0500 Subject: [PATCH 1/5] Improving backup efficiency with shutil Removed the old backup code that created a zip on a file-by-file basis and replaced it with make_archive from shutil to just zip the whole directory --- app/classes/shared/helpers.py | 11 +---------- app/classes/shared/server.py | 5 ++++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 8ee0de64..0091a647 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -553,13 +553,4 @@ class Helpers: return json.loads(content) - @staticmethod - def zip_directory(file, path, compression=zipfile.ZIP_LZMA): - with zipfile.ZipFile(file, 'w', compression) as zf: - for root, dirs, files in os.walk(path): - for file in files: - zf.write(os.path.join(root, file), - os.path.relpath(os.path.join(root, file), - os.path.join(path, '..'))) - -helper = Helpers() +helper = Helpers() \ No newline at end of file diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index ab6f6372..2fb6a004 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -10,6 +10,8 @@ import threading import schedule import logging.config import zipfile +import shutil +import zlib from app.classes.shared.helpers import helper @@ -324,7 +326,8 @@ class Server: try: backup_filename = "{}/{}.zip".format(conf['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) logger.info("Creating backup of server '{}' (ID#{}) at '{}'".format(self.settings['server_name'], self.server_id, backup_filename)) - helper.zip_directory(backup_filename, self.server_path) + # helper.zip_directory(backup_filename, self.server_path) + shutil.make_archive(backup_filename, zip, self.server_path) backup_list = self.list_backups() if len(self.list_backups()) > conf["max_backups"]: oldfile = backup_list[0] From 4757ac05735145543d77c90b5bcbc65de45d8677 Mon Sep 17 00:00:00 2001 From: xithical <86810816+xithical@users.noreply.github.com> Date: Fri, 13 Aug 2021 22:06:22 -0500 Subject: [PATCH 2/5] Repaired backup zipper and old backup removals *Fixed zip arg, should be in quotes *Set while statement to constantly check for and remove old backups *Repaired old backup removal not pulling correct path --- app/classes/shared/server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 2fb6a004..e08e9843 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -324,15 +324,15 @@ class Server: logger.info("Starting server {} (ID {}) backup".format(self.name, self.server_id)) conf = db_helper.get_backup_config(self.server_id) try: - backup_filename = "{}/{}.zip".format(conf['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) + backup_filename = "{}/{}".format(conf['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) logger.info("Creating backup of server '{}' (ID#{}) at '{}'".format(self.settings['server_name'], self.server_id, backup_filename)) - # helper.zip_directory(backup_filename, self.server_path) - shutil.make_archive(backup_filename, zip, self.server_path) - backup_list = self.list_backups() - if len(self.list_backups()) > conf["max_backups"]: + shutil.make_archive(backup_filename, 'zip', self.server_path) + while len(self.list_backups()) > conf["max_backups"]: + backup_list = self.list_backups() oldfile = backup_list[0] - logger.info("Removing old backup '{}'".format(oldfile)) - os.remove(oldfile) + oldfile_path = "{}/{}".format(conf['backup_path'], oldfile['path']) + logger.info("Removing old backup '{}'".format(oldfile['path'])) + os.remove(oldfile_path) except: logger.exception("Failed to create backup of server {} (ID {})".format(self.name, self.server_id)) From 096a38ae5435380e55a21bbcf41f59c50c332dc4 Mon Sep 17 00:00:00 2001 From: Scott R <86810816+xithical@users.noreply.github.com> Date: Fri, 13 Aug 2021 23:28:41 -0500 Subject: [PATCH 3/5] Redoing shutil zip after merging new DB things Merged from Pretzel for DB change things and needed to readd code for shutil for zipping --- app/classes/shared/helpers.py | 8 -------- app/classes/shared/server.py | 12 ++++++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 5d124d32..7242fda0 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -555,14 +555,6 @@ class Helpers: return json.loads(content) @staticmethod - def zip_directory(file, path, compression=zipfile.ZIP_LZMA): - with zipfile.ZipFile(file, 'w', compression) as zf: - for root, dirs, files in os.walk(path): - for file in files: - zf.write(os.path.join(root, file), - os.path.relpath(os.path.join(root, file), - os.path.join(path, '..'))) - @staticmethod def copy_files(source, dest): if os.path.isfile(source): shutil.copyfile(source, dest) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 2266b256..28292b98 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -11,6 +11,8 @@ import schedule import logging.config import zipfile from threading import Thread +import shutil +import zlib from app.classes.shared.helpers import helper @@ -366,15 +368,13 @@ class Server: try: backup_filename = "{}/{}.zip".format(self.settings['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) logger.info("Creating backup of server '{}' (ID#{}) at '{}'".format(self.settings['server_name'], self.server_id, backup_filename)) - helper.zip_directory(backup_filename, self.server_path) + shutil.make_archive(backup_filename, 'zip', self.server_path) while len(self.list_backups()) > conf["max_backups"]: backup_list = self.list_backups() oldfile = backup_list[0] - backup_path = self.settings['backup_path'] - old_file_name = oldfile['path'] - back_old_file = os.path.join(backup_path, old_file_name) - logger.info("Removing old backup '{}'".format(oldfile)) - os.remove(back_old_file) + oldfile_path = "{}/{}".format(conf['backup_path'], oldfile['path']) + logger.info("Removing old backup '{}'".format(oldfile['path'])) + os.remove(oldfile_path) self.is_backingup = False logger.info("Backup of server: {} completed".format(self.name)) return From 5b8764cb45dd9a17dd80f639098f228d19771fd3 Mon Sep 17 00:00:00 2001 From: Scott R <86810816+xithical@users.noreply.github.com> Date: Sat, 14 Aug 2021 00:00:09 -0500 Subject: [PATCH 4/5] Removed extra .zip Unnecessary .zip in backup filename generation in server.py, nuked it --- app/classes/shared/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 28292b98..e4261b16 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -366,7 +366,7 @@ class Server: self.is_backingup = True conf = db_helper.get_backup_config(self.server_id) try: - backup_filename = "{}/{}.zip".format(self.settings['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) + backup_filename = "{}/{}".format(self.settings['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) logger.info("Creating backup of server '{}' (ID#{}) at '{}'".format(self.settings['server_name'], self.server_id, backup_filename)) shutil.make_archive(backup_filename, 'zip', self.server_path) while len(self.list_backups()) > conf["max_backups"]: From 689e4abb518579df1e639990f2d2b3f06eb128aa Mon Sep 17 00:00:00 2001 From: Scott R <86810816+xithical@users.noreply.github.com> Date: Sat, 14 Aug 2021 00:07:24 -0500 Subject: [PATCH 5/5] Oops! All backups are gone... but not anymore! While loop automatically removed all old backups greater than the max backups threshold... including if it was set to 0. Set a test for the while loop to make sure max backups is greater than 0 before deletey delete --- app/classes/shared/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index e4261b16..f28546c6 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -369,7 +369,7 @@ class Server: backup_filename = "{}/{}".format(self.settings['backup_path'], datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) logger.info("Creating backup of server '{}' (ID#{}) at '{}'".format(self.settings['server_name'], self.server_id, backup_filename)) shutil.make_archive(backup_filename, 'zip', self.server_path) - while len(self.list_backups()) > conf["max_backups"]: + while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0: backup_list = self.list_backups() oldfile = backup_list[0] oldfile_path = "{}/{}".format(conf['backup_path'], oldfile['path'])