Merge branch 'backup-fixes' into 'dev'

Rework of files handling and added exclusions selection for backups.

See merge request crafty-controller/crafty-commander!174

app/classes/controllers/management_controller.py

@@ -104,5 +104,17 @@ class Management_Controller:
         return management_helper.get_backup_config(server_id)
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None):
-        return management_helper.set_backup_config(server_id, backup_path, max_backups)
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None):
+        return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs)
+
+    @staticmethod
+    def get_excluded_backup_dirs(server_id: int):
+        return management_helper.get_excluded_backup_dirs(server_id)
+
+    @staticmethod
+    def add_excluded_backup_dir(server_id: int, dir_to_add: str):
+        management_helper.add_excluded_backup_dir(server_id, dir_to_add)
+
+    @staticmethod
+    def del_excluded_backup_dir(server_id: int, dir_to_del: str):
+        management_helper.del_excluded_backup_dir(server_id, dir_to_del)

app/classes/models/management.py

@@ -125,7 +125,7 @@ class Schedules(Model):
 #                                   Backups Class
 #************************************************************************************************
 class Backups(Model):
-    directories = CharField(null=True)
+    excluded_dirs = CharField(null=True)
     max_backups = IntegerField()
     server_id = ForeignKeyField(Servers, backref='backups_server')
     class Meta:
@@ -311,34 +311,37 @@ class helpers_management:
             row = Backups.select().where(Backups.server_id == server_id).join(Servers)[0]
             conf = {
                 "backup_path": row.server_id.backup_path,
-                "directories": row.directories,
+                "excluded_dirs": row.excluded_dirs,
                 "max_backups": row.max_backups,
                 "server_id": row.server_id.server_id
             }
         except IndexError:
             conf = {
                 "backup_path": None,
-                "directories": None,
+                "excluded_dirs": None,
                 "max_backups": 0,
                 "server_id": server_id
             }
         return conf
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None):
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None):
         logger.debug(f"Updating server {server_id} backup config with {locals()}")
         if Backups.select().where(Backups.server_id == server_id).count() != 0:
             new_row = False
             conf = {}
         else:
             conf = {
-                "directories": None,
+                "excluded_dirs": None,
                 "max_backups": 0,
                 "server_id":   server_id
             }
             new_row = True
         if max_backups is not None:
             conf['max_backups'] = max_backups
+        if excluded_dirs is not None:
+            dirs_to_exclude = ",".join(excluded_dirs)
+            conf['excluded_dirs'] = dirs_to_exclude
         if not new_row:
             with database.atomic():
                 if backup_path is not None:
@@ -355,5 +358,31 @@ class helpers_management:
                 Backups.create(**conf)
             logger.debug("Creating new backup record.")
 
+    def get_excluded_backup_dirs(self, server_id: int):
+        excluded_dirs = self.get_backup_config(server_id)['excluded_dirs']
+        if excluded_dirs is not None and excluded_dirs != "":
+            dir_list = excluded_dirs.split(",")
+        else:
+            dir_list = []
+        return dir_list
+
+    def add_excluded_backup_dir(self, server_id: int, dir_to_add: str):
+        dir_list = self.get_excluded_backup_dirs()
+        if dir_to_add not in dir_list:
+            dir_list.append(dir_to_add)
+            excluded_dirs = ",".join(dir_list)
+            self.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs)
+        else:
+            logger.debug(f"Not adding {dir_to_add} to excluded directories - already in the excluded directory list for server ID {server_id}")
+
+    def del_excluded_backup_dir(self, server_id: int, dir_to_del: str):
+        dir_list = self.get_excluded_backup_dirs()
+        if dir_to_del in dir_list:
+            dir_list.remove(dir_to_del)
+            excluded_dirs = ",".join(dir_list)
+            self.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs)
+        else:
+            logger.debug(f"Not removing {dir_to_del} from excluded directories - not in the excluded directory list for server ID {server_id}")
+
 
 management_helper = helpers_management()

app/classes/models/servers.py

@@ -157,10 +157,12 @@ class helper_servers:
     def get_all_servers_stats():
         servers = servers_helper.get_all_defined_servers()
         server_data = []
-
-        for s in servers:
-            latest = Server_Stats.select().where(Server_Stats.server_id == s.get('server_id')).order_by(Server_Stats.created.desc()).limit(1)
-            server_data.append({'server_data': s, "stats": db_helper.return_rows(latest)[0], "user_command_permission":True})
+        try:
+            for s in servers:
+                latest = Server_Stats.select().where(Server_Stats.server_id == s.get('server_id')).order_by(Server_Stats.created.desc()).limit(1)
+                server_data.append({'server_data': s, "stats": db_helper.return_rows(latest)[0], "user_command_permission":True})
+        except IndexError as ex:
+            logger.error(f"Stats collection failed with error: {ex}. Was a server just created?")
         return server_data
 
     @staticmethod

app/classes/shared/file_helpers.py

@@ -0,0 +1,90 @@
+import os
+import shutil
+import sys
+import logging
+import pathlib
+
+from app.classes.shared.console import console
+
+logger = logging.getLogger(__name__)
+
+try:
+    from zipfile import ZipFile
+
+except ModuleNotFoundError as err:
+    logger.critical(f"Import Error: Unable to load {err.name} module", exc_info=True)
+    console.critical(f"Import Error: Unable to load {err.name} module")
+    sys.exit(1)
+
+class FileHelpers:
+    allowed_quotes = [
+        "\"",
+        "'",
+        "`"
+     ]
+
+    def del_dirs(self, path):
+        path = pathlib.Path(path)
+        for sub in path.iterdir():
+            if sub.is_dir():
+                # Delete folder if it is a folder
+                self.del_dirs(sub)
+            else:
+                # Delete file if it is a file:
+                sub.unlink()
+
+        # This removes the top-level folder:
+        path.rmdir()
+        return True
+
+    @staticmethod
+    def del_file(path):
+        path = pathlib.Path(path)
+        try:
+            logger.debug(f"Deleting file: {path}")
+            #Remove the file
+            os.remove(path)
+            return True
+        except FileNotFoundError:
+            logger.error(f"Path specified is not a file or does not exist. {path}")
+            return False
+
+    @staticmethod
+    def copy_dir(src_path, dest_path, dirs_exist_ok=False):
+        # pylint: disable=unexpected-keyword-arg
+        shutil.copytree(src_path, dest_path, dirs_exist_ok=dirs_exist_ok)
+
+    @staticmethod
+    def copy_file(src_path, dest_path):
+        shutil.copy(src_path, dest_path)
+
+    def move_dir(self, src_path, dest_path):
+        self.copy_dir(src_path, dest_path)
+        self.del_dirs(src_path)
+
+    def move_file(self, src_path, dest_path):
+        self.copy_file(src_path, dest_path)
+        self.del_file(src_path)
+
+    @staticmethod
+    def make_archive(path_to_destination, path_to_zip):
+        # create a ZipFile object
+        path_to_destination += '.zip'
+        with ZipFile(path_to_destination, 'w') as z:
+            for root, _dirs, files in os.walk(path_to_zip, topdown=True):
+                ziproot = path_to_zip
+                for file in files:
+                    try:
+                        logger.info(f"backing up: {os.path.join(root, file)}")
+                        if os.name == "nt":
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, ""), file))
+                        else:
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, "/"), file))
+
+                    except Exception as e:
+                        logger.warning(f"Error backing up: {os.path.join(root, file)}! - Error was: {e}")
+
+
+        return True
+
+file_helper = FileHelpers()

app/classes/shared/helpers.py

@@ -13,7 +13,6 @@ import logging
 import html
 import zipfile
 import pathlib
-import shutil
 import ctypes
 from datetime import datetime
 from socket import gethostname
@@ -22,6 +21,7 @@ from requests import get
 
 from app.classes.web.websocket_helper import websocket_helper
 from app.classes.shared.console import console
+from app.classes.shared.file_helpers import file_helper
 
 logger = logging.getLogger(__name__)
 
@@ -369,7 +369,7 @@ class Helpers:
 
                 for item in os.listdir(full_root_path):
                     try:
-                        shutil.move(os.path.join(full_root_path, item), os.path.join(new_dir, item))
+                        file_helper.move_dir(os.path.join(full_root_path, item), os.path.join(new_dir, item))
                     except Exception as ex:
                         logger.error(f'ERROR IN ZIP IMPORT: {ex}')
             except Exception as ex:
@@ -773,6 +773,12 @@ class Helpers:
                 websocket_helper.broadcast_user(user_id, 'send_temp_path',{
                 'path': tempDir
                 })
+    @staticmethod
+    def backup_select(path, user_id):
+        if user_id:
+            websocket_helper.broadcast_user(user_id, 'send_temp_path',{
+            'path': path
+        })
 
     @staticmethod
     def unzip_backup_archive(backup_path, zip_name):
@@ -804,7 +810,7 @@ class Helpers:
     @staticmethod
     def copy_files(source, dest):
         if os.path.isfile(source):
-            shutil.copyfile(source, dest)
+            file_helper.copy_file(source, dest)
             logger.info("Copying jar %s to %s", source, dest)
         else:
             logger.info("Source jar does not exist.")

app/classes/shared/main_controller.py

@@ -2,7 +2,6 @@ import os
 import pathlib
 import time
 import logging
-import shutil
 import tempfile
 from distutils import dir_util
 from typing import Union
@@ -23,6 +22,7 @@ from app.classes.models.servers import servers_helper
 from app.classes.shared.console import console
 from app.classes.shared.helpers import helper
 from app.classes.shared.server import Server
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.minecraft.server_props import ServerProps
 from app.classes.minecraft.serverjars import server_jar_obj
@@ -153,13 +153,13 @@ class Controller:
             final_path = os.path.join(server_path, str(server['server_name']))
             os.mkdir(final_path)
             try:
-                shutil.copy(server['log_path'], final_path)
+                file_helper.copy_file(server['log_path'], final_path)
             except Exception as e:
                 logger.warning(f"Failed to copy file with error: {e}")
         #Copy crafty logs to archive dir
         full_log_name = os.path.join(crafty_path, 'logs')
-        shutil.copytree(os.path.join(self.project_root, 'logs'), full_log_name)
-        shutil.make_archive(tempZipStorage, "zip", tempDir)
+        file_helper.copy_dir(os.path.join(self.project_root, 'logs'), full_log_name)
+        file_helper.make_archive(tempZipStorage, tempDir)
 
         tempZipStorage += '.zip'
         websocket_helper.broadcast_user(exec_user['user_id'], 'send_logs_bootbox', {
@@ -374,7 +374,7 @@ class Controller:
             if str(item) == 'server.properties':
                 has_properties = True
             try:
-                shutil.move(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                file_helper.move_file(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
             except Exception as ex:
                 logger.error(f'ERROR IN ZIP IMPORT: {ex}')
         if not has_properties:
@@ -462,7 +462,7 @@ class Controller:
             if str(item) == 'server.properties':
                 has_properties = True
             try:
-                shutil.move(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                file_helper.move_file(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
             except Exception as ex:
                 logger.error(f'ERROR IN ZIP IMPORT: {ex}')
         if not has_properties:
@@ -558,11 +558,11 @@ class Controller:
                     self.stop_server(server_id)
                 if files:
                     try:
-                        shutil.rmtree(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['path']))
+                        file_helper.del_dirs(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['path']))
                     except Exception as e:
                         logger.error(f"Unable to delete server files for server with ID: {server_id} with error logged: {e}")
                     if helper.check_path_exists(self.servers.get_server_data_by_id(server_id)['backup_path']):
-                        shutil.rmtree(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['backup_path']))
+                        file_helper.del_dirs(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['backup_path']))
 
 
                 #Cleanup scheduled tasks

app/classes/shared/server.py

@@ -5,9 +5,9 @@ import time
 import datetime
 import threading
 import logging.config
-import shutil
 import subprocess
 import html
+import tempfile
 from apscheduler.schedulers.background import BackgroundScheduler
 #TZLocal is set as a hidden import on win pipeline
 from tzlocal import get_localzone
@@ -20,6 +20,7 @@ from app.classes.models.server_permissions import server_permissions
 from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
 from app.classes.shared.translation import translation
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.web.websocket_helper import websocket_helper
 
@@ -580,14 +581,35 @@ class Server:
             backup_filename = f"{self.settings['backup_path']}/{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
             logger.info(f"Creating backup of server '{self.settings['server_name']}'" +
                         f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'")
-            shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', self.server_path)
+
+            tempDir = tempfile.mkdtemp()
+            # pylint: disable=unexpected-keyword-arg
+            file_helper.copy_dir(self.server_path, tempDir, dirs_exist_ok=True)
+            excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id)
+            server_dir = helper.get_os_understandable_path(self.settings['path'])
+
+            for my_dir in excluded_dirs:
+                # Take the full path of the excluded dir and replace the server path with the temp path
+                # This is so that we're only deleting excluded dirs from the temp path and not the server path
+                excluded_dir = helper.get_os_understandable_path(my_dir).replace(server_dir, helper.get_os_understandable_path(tempDir))
+                # Next, check to see if it is a directory
+                if os.path.isdir(excluded_dir):
+                    # If it is a directory, recursively delete the entire directory from the backup
+                    file_helper.del_dirs(excluded_dir)
+                else:
+                    # If not, just remove the file
+                    os.remove(excluded_dir)
+            file_helper.make_archive(helper.get_os_understandable_path(backup_filename), tempDir)
+
             while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0:
                 backup_list = self.list_backups()
                 oldfile = backup_list[0]
                 oldfile_path = f"{conf['backup_path']}/{oldfile['path']}"
                 logger.info(f"Removing old backup '{oldfile['path']}'")
                 os.remove(helper.get_os_understandable_path(oldfile_path))
+
             self.is_backingup = False
+            file_helper.del_dirs(tempDir)
             logger.info(f"Backup of server: {self.name} completed")
             return
         except:

app/classes/web/ajax_handler.py

@@ -99,6 +99,141 @@ class AjaxHandler(BaseHandler):
                         helper.generate_zip_dir(path))
             self.finish()
 
+        elif page == "get_backup_tree":
+            server_id = self.get_argument('id', None)
+            folder = self.get_argument('path', None)
+
+            output = ""
+
+            file_list = os.listdir(folder)
+            file_list = sorted(file_list, key=str.casefold)
+            output += \
+        f"""<ul class="tree-nested d-block" id="{folder}ul">"""\
+
+            for raw_filename in file_list:
+                filename = html.escape(raw_filename)
+                rel = os.path.join(folder, raw_filename)
+                dpath = os.path.join(folder, filename)
+                if str(dpath) in self.controller.management.get_excluded_backup_dirs(server_id):
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" class="checkBoxClass" name="root_path" value="{dpath}" checked>
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>
+                            \n"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' class="checkBoxClass" name='root_path' value="{dpath}" checked><span style="margin-right: 6px;">
+                        <i class="far fa-file"></i></span></input>{filename}</li>"""
+
+                else:
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" class="checkBoxClass" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>
+                            \n"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' class="checkBoxClass" name='root_path' value="{dpath}">
+                        <span style="margin-right: 6px;"><i class="far fa-file"></i></span></input>{filename}</li>"""
+            self.write(helper.get_os_understandable_path(folder) + '\n' +
+                        output)
+            self.finish()
+
+        elif page == "get_backup_dir":
+            server_id = self.get_argument('id', None)
+            folder = self.get_argument('path', None)
+            output = ""
+
+            file_list = os.listdir(folder)
+            file_list = sorted(file_list, key=str.casefold)
+            output += \
+        f"""<ul class="tree-nested d-block" id="{folder}ul">"""\
+
+            for raw_filename in file_list:
+                filename = html.escape(raw_filename)
+                rel = os.path.join(folder, raw_filename)
+                dpath = os.path.join(folder, filename)
+                if str(dpath) in self.controller.management.get_excluded_backup_dirs(server_id):
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' name='root_path' value='{dpath}'><span style="margin-right: 6px;">
+                        <i class="far fa-file"></i></span></input>{filename}</li>"""
+
+                else:
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' name='root_path' value='{dpath}'>
+                        <span style="margin-right: 6px;"><i class="far fa-file"></i></span></input>{filename}</li>"""
+
+            self.write(helper.get_os_understandable_path(folder) + '\n' +
+                        output)
+            self.finish()
+
+        elif page == "get_dir":
+            server_id = self.get_argument('id', None)
+            path = self.get_argument('path', None)
+
+            if not self.check_server_id(server_id, 'get_tree'):
+                return
+            else:
+                server_id = bleach.clean(server_id)
+
+            if helper.validate_traversal(self.controller.servers.get_server_data_by_id(server_id)['path'], path):
+                self.write(helper.get_os_understandable_path(path) + '\n' +
+                        helper.generate_dir(path))
+            self.finish()
 
     @tornado.web.authenticated
     def post(self, page):
@@ -199,6 +334,11 @@ class AjaxHandler(BaseHandler):
             helper.unzipServer(path, exec_user['user_id'])
             return
 
+        elif page == "backup_select":
+            path = self.get_argument('path', None)
+            helper.backup_select(path, exec_user['user_id'])
+            return
+
 
     @tornado.web.authenticated
     def delete(self, page):

app/classes/web/file_handler.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 import logging
 import tornado.web
 import tornado.escape
@@ -7,6 +6,7 @@ import bleach
 
 from app.classes.shared.console import console
 from app.classes.shared.helpers import helper
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.web.base_handler import BaseHandler
 from app.classes.models.server_permissions import Enum_Permissions_Server
@@ -232,7 +232,7 @@ class FileHandler(BaseHandler):
                 return
 
             # Delete the file
-            os.remove(file_path)
+            file_helper.del_file(file_path)
 
         elif page == "del_dir":
             if not permissions['Files'] in user_perms:
@@ -258,7 +258,8 @@ class FileHandler(BaseHandler):
             # Delete the directory
             # os.rmdir(dir_path)     # Would only remove empty directories
             if helper.validate_traversal(helper.get_os_understandable_path(server_info['path']), dir_path):
-                shutil.rmtree(dir_path)  # Removes also when there are contents
+                # Removes also when there are contents
+                file_helper.del_dirs(dir_path)
 
     @tornado.web.authenticated
     def put(self, page):

app/classes/web/panel_handler.py

@@ -204,6 +204,29 @@ class PanelHandler(BaseHandler):
                 exec_user_role.add(role['role_name'])
             defined_servers = self.controller.servers.get_authorized_servers(exec_user["user_id"])
 
+
+        user_order = self.controller.users.get_user_by_id(exec_user['user_id'])
+        user_order = user_order['server_order'].split(',')
+        page_servers = []
+        server_ids = []
+
+        for server_id in user_order:
+            for server in defined_servers:
+                if str(server['server_id']) == str(server_id):
+                    page_servers.append(server)
+
+
+        for server in defined_servers:
+            server_ids.append(str(server['server_id']))
+            if server not in page_servers:
+                page_servers.append(server)
+        for server_id in user_order:
+            #remove IDs in list that user no longer has access to
+            if str(server_id) not in server_ids:
+                user_order.remove(server_id)
+        defined_servers = page_servers
+
+
         page_data: Dict[str, Any] = {
             # todo: make this actually pull and compare version data
             'update_available': False,
@@ -428,6 +451,15 @@ class PanelHandler(BaseHandler):
                         return
                 server_info = self.controller.servers.get_server_data_by_id(server_id)
                 page_data['backup_config'] = self.controller.management.get_backup_config(server_id)
+                exclusions = []
+                page_data['exclusions'] = self.controller.management.get_excluded_backup_dirs(server_id)
+                #makes it so relative path is the only thing shown
+                for file in page_data['exclusions']:
+                    if helper.is_os_windows():
+                        exclusions.append(file.replace(server_info['path']+'\\', ""))
+                    else:
+                        exclusions.append(file.replace(server_info['path']+'/', ""))
+                page_data['exclusions'] = exclusions
                 self.controller.refresh_server_settings(server_id)
                 try:
                     page_data['backup_list'] = server.list_backups()
@@ -1027,6 +1059,11 @@ class PanelHandler(BaseHandler):
             logger.debug(self.request.arguments)
             server_id = self.get_argument('id', None)
             server_obj = self.controller.servers.get_server_obj(server_id)
+            check_changed = self.get_argument('changed')
+            if str(check_changed) == str(1):
+                checked = self.get_body_arguments('root_path')
+            else:
+                checked = self.controller.management.get_excluded_backup_dirs(server_id)
             if superuser:
                 backup_path = bleach.clean(self.get_argument('backup_path', None))
                 if helper.is_os_windows():
@@ -1052,7 +1089,7 @@ class PanelHandler(BaseHandler):
             server_obj = self.controller.servers.get_server_obj(server_id)
             server_obj.backup_path = backup_path
             self.controller.servers.update_server(server_obj)
-            self.controller.management.set_backup_config(server_id, max_backups=max_backups)
+            self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked)
 
             self.controller.management.add_to_audit_log(exec_user['user_id'],
                                        f"Edited server {server_id}: updated backups",

app/classes/web/server_handler.py

@@ -2,12 +2,12 @@ import sys
 import json
 import logging
 import os
-import shutil
 import libgravatar
 import requests
 
 from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
+from app.classes.shared.file_helpers import file_helper
 from app.classes.web.base_handler import BaseHandler
 from app.classes.models.crafty_permissions import Enum_Permissions_Crafty
 from app.classes.minecraft.serverjars import server_jar_obj
@@ -169,7 +169,7 @@ class ServerHandler(BaseHandler):
                     new_server_path = os.path.join(helper.servers_dir, new_server_uuid)
 
                     # copy the old server
-                    shutil.copytree(server_data.get('path'), new_server_path)
+                    file_helper.copy_dir(server_data.get('path'), new_server_path)
 
                     # TODO get old server DB data to individual variables
                     stop_command = server_data.get('stop_command')
@@ -177,7 +177,7 @@ class ServerHandler(BaseHandler):
                     new_executable = server_data.get('executable')
                     new_server_log_file = str(helper.get_os_understandable_path(server_data.get('log_path'))).replace(server_uuid, new_server_uuid)
                     server_port = server_data.get('server_port')
-                    server_type = server_data.get('server_type')
+                    server_type = server_data.get('type')
 
                     self.controller.servers.create_server(new_server_name,
                                                           new_server_uuid,
@@ -250,7 +250,7 @@ class ServerHandler(BaseHandler):
                                            new_server_id,
                                            self.get_remote_ip())
                 #deletes temp dir
-                shutil.rmtree(zip_path)
+                file_helper.del_dirs(zip_path)
             else:
                 if len(server_parts) != 2:
                     self.redirect("/panel/error?error=Invalid server data")
@@ -333,7 +333,7 @@ class ServerHandler(BaseHandler):
                                            new_server_id,
                                            self.get_remote_ip())
                 #deletes temp dir
-                shutil.rmtree(zip_path)
+                file_helper.del_dirs(zip_path)
             else:
                 if len(server_parts) != 2:
                     self.redirect("/panel/error?error=Invalid server data")

app/frontend/templates/panel/parts/details_stats.html

@@ -136,15 +136,8 @@
 
     /* TODO Update each element */
     if (server.running){
-      if (server['stats']['waiting_start']){
-        server_status.setAttribute("class", "text-warning");
-        server_status.innerHTML = `{{ translate('serverStats', 'starting', data['lang']) }}`;
-      }
-      else
-      {
         server_status.setAttribute("class", "text-success");
         server_status.innerHTML = `{{ translate('serverStats', 'online', data['lang']) }}`;
-      }
 
       startedUTC = server.started;
       startedUTC = moment.utc(startedUTC, 'YYYY-MM-DD HH:mm:ss');

app/frontend/templates/panel/server_backup.html

@@ -35,6 +35,8 @@
 
             <div class="row">
                 <div class="col-md-6 col-sm-12">
+                  <br>
+                  <br>
                     <form class="forms-sample" method="post" action="/panel/server_backup">
                       {% raw xsrf_form_html() %}
                       <input type="hidden" name="id" value="{{ data['server_stats']['server_id']['server_id'] }}">
@@ -54,6 +56,39 @@
                         <label for="server_path">{{ translate('serverBackups', 'maxBackups', data['lang']) }} <small class="text-muted ml-1"> - {{ translate('serverBackups', 'maxBackupsDesc', data['lang']) }}</small> </label>
                         <input type="text" class="form-control" name="max_backups" id="max_backups" value="{{ data['backup_config']['max_backups']  }}" placeholder="{{ translate('serverBackups', 'maxBackups', data['lang']) }}" >
                       </div>
+                        <div class="form-group">
+                          <label for="server">{{ translate('serverBackups', 'exclusionsTitle', data['lang']) }} <small> - {{ translate('serverBackups', 'excludedChoose', data['lang']) }}</small></label>
+                          <br>
+                          <button class="btn btn-primary mr-2" id="root_files_button" data-server_path="{{ data['server_stats']['server_id']['path']}}" type="button">{{ translate('serverBackups', 'clickExclude', data['lang']) }}</button>
+                        </div>
+                        <input type="number" class="form-control" name="changed" id="changed" value="0" style="visibility: hidden;"></input>
+                      <div class="modal fade" id="dir_select" tabindex="-1" role="dialog" aria-labelledby="dir_select" aria-hidden="true">
+                        <div class="modal-dialog" role="document">
+                          <div class="modal-content">
+                            <div class="modal-header">
+                              <h5 class="modal-title" id="exampleModalLongTitle">{{ translate('serverBackups', 'excludedChoose', data['lang']) }}</h5>
+                              <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                                <span aria-hidden="true">&times;</span>
+                              </button>
+                            </div>
+                            <div class="modal-body">
+                              <div class="tree-ctx-item" id="main-tree-div" data-path="" style="overflow: scroll; max-height:75%;">
+                                <input type="checkbox" id="main-tree-input" name="root_path" value="" disabled>
+                                  <span id="main-tree" class="files-tree-title tree-caret-down root-dir" data-path="">
+                                    <i class="far fa-folder"></i>
+                                    <i class="far fa-folder-open"></i>
+                                    {{ translate('serverFiles', 'files', data['lang']) }}
+                                    </span>
+                                </input>
+                              </div>
+                            </div>
+                            <div class="modal-footer">
+                              <button type="button" id="modal-cancel" class="btn btn-secondary" data-dismiss="modal">{{ translate('serverBackups', 'cancel', data['lang']) }}</button>
+                              <button type="button" id="modal-okay" data-dismiss="modal" class="btn btn-primary">{{ translate('serverWizard', 'save', data['lang']) }}</button>
+                            </div>
+                          </div>
+                        </div>
+                      </div>
 
                       <button type="submit" class="btn btn-success mr-2">{{ translate('serverBackups', 'save', data['lang']) }}</button>
                       <button type="reset" class="btn btn-light">{{ translate('serverBackups', 'cancel', data['lang']) }}</button>
@@ -61,15 +96,10 @@
                 </div>
 
                 <div class="col-md-6 col-sm-12">
-                    <div class="card">
-                      <div class="card-body">
-                        <h4 class="card-title">{{ translate('serverBackups', 'currentBackups', data['lang']) }}</h4>
-
-                      </div>
-                    </div>
                     <div class="text-center">
 
                       <table class="table table-responsive dataTable" id="backup_table">
+                        <h4 class="card-title">{{ translate('serverBackups', 'currentBackups', data['lang']) }}</h4>
                         <thead>
                         <tr>
                             <th width="10%">{{ translate('serverBackups', 'options', data['lang']) }}</th>
@@ -107,6 +137,20 @@
                     </div>
                 </div>
             </div>
+            <div class="col-md-12 col-sm-12">
+              <br>
+              <br>
+              <div class="card-header header-sm d-flex justify-content-between align-items-center">
+                <h4 class="card-title"><i class="fas fa-server"></i> {{ translate('serverBackups', 'excludedBackups', data['lang']) }}  <small class="text-muted ml-1"></small> </h4>
+              </div>
+              <br>
+              <ul>
+                {% for item in data['exclusions'] %}
+                <li>{{item}}</li>
+                <br>
+                {% end %}
+              </ul>
+            </div>
 
         </div>
       </div>
@@ -116,6 +160,44 @@
 
 
 </div>
+<style>
+  /* Remove default bullets */
+ .tree-view,
+ .tree-nested {
+   list-style-type: none;
+   margin: 0;
+   padding: 0;
+   margin-left: 10px;
+ }
+
+ /* Style the items */
+ .tree-item,
+ .files-tree-title {
+   cursor: pointer;
+   user-select: none; /* Prevent text selection */
+ }
+
+ /* Create the caret/arrow with a unicode, and style it */
+ .tree-caret .fa-folder {
+   display: inline-block;
+ }
+ .tree-caret .fa-folder-open {
+   display: none;
+ }
+
+ /* Rotate the caret/arrow icon when clicked on (using JavaScript) */
+ .tree-caret-down .fa-folder {
+   display: none;
+ }
+ .tree-caret-down .fa-folder-open {
+   display: inline-block;
+ }
+
+ /* Hide the nested list */
+ .tree-nested {
+   display: none;
+ }
+</style>
 <!-- content-wrapper ends -->
 
 {% end %}
@@ -123,7 +205,7 @@
 {% block js %}
 <script>
 
-  const serverId = new URLSearchParams(document.location.search).get('id')
+const server_id = new URLSearchParams(document.location.search).get('id')
 
 
   //used to get cookies from browser - this is part of tornados xsrf protection - it's for extra security
@@ -228,7 +310,7 @@
           console.log(result);
           if (result == true) {
             var full_path = backup_path + '/' + file_to_del;
-            del_backup(full_path, serverId);
+            del_backup(full_path, server_id);
           }
         }
       });
@@ -246,13 +328,14 @@
             label: '<i class="fas fa-times"></i> {{ translate("serverBackups", "cancel", data['lang']) }}'
           },
           confirm: {
-            label: '<i class="fas fa-check"></i> {{ translate("serverBackups", "restore", data['lang']) }}'
+            label: '<i class="fas fa-check"></i> {{ translate("serverBackups", "restore", data['lang']) }}',
+          className: 'btn-outline-danger'
           }
         },
         callback: function (result) {
           console.log(result);
           if (result == true) {
-            restore_backup(file_to_restore, serverId);
+            restore_backup(file_to_restore, server_id);
           }
         }
       });
@@ -260,7 +343,141 @@
 
   });
 
+document.getElementById("modal-cancel").addEventListener("click", function(){
+  document.getElementById("root_files_button").classList.remove('clicked');
+  document.getElementById("main-tree-div").innerHTML = '<input type="checkbox" id="main-tree-input" name="root_path" value="" disabled><span id="main-tree" class="files-tree-title tree-caret-down root-dir" data-path=""><i class="far fa-folder"></i><i class="far fa-folder-open"></i>{{ translate("serverFiles", "files", data["lang"]) }}</span></input>'
+})
+
+  document.getElementById("root_files_button").addEventListener("click", function(){
+    if($("#root_files_button").data('server_path') != ""){
+      if(document.getElementById('root_files_button').classList.contains('clicked')){
+        show_file_tree();
+        return;
+      }else{
+        document.getElementById('root_files_button').classList.add('clicked');
+        document.getElementById("changed").value = 1;
+      }
+    path = $("#root_files_button").data('server_path')
+    console.log($("#root_files_button").data('server_path'))
+    var token = getCookie("_xsrf");
+    var dialog = bootbox.dialog({
+      message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
+      closeButton: false
+  });
+
+    $.ajax({
+      type: "POST",
+      headers: {'X-XSRFToken': token},
+      url: '/ajax/backup_select?id='+server_id+'&path='+path,
+    });
+  }else{
+    bootbox.alert("You must input a path before selecting this button");
+  }
+  });
+  if (webSocket) {
+    webSocket.on('send_temp_path', function (data) {
+      setTimeout(function(){
+        var x = document.querySelector('.bootbox');
+        if (x) {
+          x.remove()
+        }
+        var x = document.querySelector('.modal-backdrop');
+        if (x) {
+          x.remove()
+        }
+        document.getElementById('main-tree-input').setAttribute('value', data.path)
+        getTreeView(data.path);
+        show_file_tree();
+
+    }, 5000);
+    });
+  }
+
+  function getTreeView(path) {
+    path = path
+
+    $.ajax({
+      type: "GET",
+      url: '/ajax/get_backup_tree?id='+server_id+'&path='+path,
+      dataType: 'text',
+      success: function(data){
+        console.log("got response:");
+        console.log(data);
+
+        dataArr = data.split('\n');
+        serverDir = dataArr.shift(); // Remove & return first element (server directory)
+        text = dataArr.join('\n');
+
+        try{
+          document.getElementById('main-tree-div').innerHTML += text;
+          document.getElementById('main-tree').parentElement.classList.add("clicked");
+        }catch{
+          document.getElementById('files-tree').innerHTML = text;
+        }
+        
+
+        document.getElementsByClassName('files-tree-title')[0].setAttribute('data-path', serverDir);
+        document.getElementsByClassName('files-tree-title')[0].setAttribute('data-name', 'Files');
+
+      },
+    });
+  }
+  function getToggleMain(event) {
+    path = event.target.parentElement.getAttribute('data-path');
+        document.getElementById("files-tree").classList.toggle("d-block");
+        document.getElementById(path+"span").classList.toggle("tree-caret-down");
+        document.getElementById(path+"span").classList.toggle("tree-caret");
+    }
+  
+
+  function getDirView(event) {
+      path = event.target.parentElement.getAttribute('data-path');
+
+      if (document.getElementById(path).classList.contains('clicked')){
+
+      var toggler = document.getElementById(path+"span");
+
+      if (toggler.classList.contains('files-tree-title')){
+          document.getElementById(path+"ul").classList.toggle("d-block");
+          document.getElementById(path+"span").classList.toggle("tree-caret-down");
+      }
+      return;
+    }else{
+    $.ajax({
+      type: "GET",
+      url: '/ajax/get_backup_dir?id='+server_id+'&path='+path,
+      dataType: 'text',
+      success: function(data){
+        console.log("got response:");
+
+        dataArr = data.split('\n');
+        serverDir = dataArr.shift(); // Remove & return first element (server directory)
+        text = dataArr.join('\n');
+
+        try{
+          document.getElementById(path+"span").classList.add('tree-caret-down');
+          document.getElementById(path).innerHTML += text;
+          document.getElementById(path).classList.add("clicked");
+        }catch{
+          console.log("Bad")
+        }
+
+        var toggler = document.getElementById(path);
+
+          if (toggler.classList.contains('files-tree-title')){
+          document.getElementById(path+"span").addEventListener("click", function caretListener() {
+            document.getElementById(path+"ul").classList.toggle("d-block");
+            document.getElementById(path+"span").classList.toggle("tree-caret-down");
+          });
+        }
+      },
+    });
+  }
+  }
+  function show_file_tree(){
+    $("#dir_select").modal();
+  }
 
 </script>
 
-{% end %}
+{% end %}

app/frontend/templates/server/bedrock_wizard.html

@@ -310,9 +310,6 @@
       message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
       closeButton: false
   });
-  setTimeout(function(){
-    dialog.modal('hide');
-}, 5000);
     $.ajax({
       type: "POST",
       headers: {'X-XSRFToken': token},
@@ -433,9 +430,20 @@ function hide(event) {
     }
     if (webSocket) {
       webSocket.on('send_temp_path', function (data) {
-        document.getElementById('main-tree-input').setAttribute('value', data.path)
-        getTreeView(data.path);
-        show_file_tree();
+        setTimeout(function(){
+          var x = document.querySelector('.bootbox');
+          if (x) {
+            x.remove()
+          }
+          var x = document.querySelector('.modal-backdrop');
+          if (x) {
+            x.remove()
+          }
+          document.getElementById('main-tree-input').setAttribute('value', data.path)
+          getTreeView(data.path);
+          show_file_tree();
+  
+      }, 5000);
       });
     }
 

app/frontend/templates/server/wizard.html

@@ -437,9 +437,6 @@
       message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
       closeButton: false
   });
-  setTimeout(function(){
-    dialog.modal('hide');
-}, 5000);
     $.ajax({
       type: "POST",
       headers: {'X-XSRFToken': token},
@@ -587,9 +584,20 @@ function hide(event) {
     }
     if (webSocket) {
       webSocket.on('send_temp_path', function (data) {
-        document.getElementById('main-tree-input').setAttribute('value', data.path)
-        getTreeView(data.path);
-        show_file_tree();
+        setTimeout(function(){
+          var x = document.querySelector('.bootbox');
+          if (x) {
+            x.remove()
+          }
+          var x = document.querySelector('.modal-backdrop');
+          if (x) {
+            x.remove()
+          }
+          document.getElementById('main-tree-input').setAttribute('value', data.path)
+          getTreeView(data.path);
+          show_file_tree();
+  
+      }, 5000);
       });
     }
 

app/migrations/20220227162446_backup_options.py

@@ -0,0 +1,8 @@
+# Generated by database migrator
+import peewee
+
+def migrate(migrator, db):
+    migrator.rename_column('backups', 'directories', 'excluded_dirs')
+
+def rollback(migrator, db):
+    migrator.rename_column('backups', 'excluded_dirs', 'directories')

app/translations/en_EN.json

@@ -236,7 +236,11 @@
         "options": "Options",
         "restoring": "Restoring Backup. This may take a while. Please be patient.",
         "restore": "Restore",
-        "confirmRestore": "Are you sure you want to restore from this backup. All current server files will changed to backup state and will be unrecoverable."
+        "confirmRestore": "Are you sure you want to restore from this backup. All current server files will changed to backup state and will be unrecoverable.",
+        "excludedBackups": "Excluded Paths: ",
+        "excludedChoose": "Choose the paths you wish to exclude from your backups",
+        "clickExclude": "Click to select Exclusions",
+        "exclusionsTitle": "Backup Exclusions"
     },
     "serverFiles": {
         "noscript": "The file manager does not work without JavaScript",