Merge branch 'dev' into devops/userns-rootless-container

[RESOLVED CONFLICTS]

app/classes/controllers/management_controller.py

@@ -104,5 +104,17 @@ class Management_Controller:
         return management_helper.get_backup_config(server_id)
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None):
-        return management_helper.set_backup_config(server_id, backup_path, max_backups)
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False,):
+        return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs, compress)
+
+    @staticmethod
+    def get_excluded_backup_dirs(server_id: int):
+        return management_helper.get_excluded_backup_dirs(server_id)
+
+    @staticmethod
+    def add_excluded_backup_dir(server_id: int, dir_to_add: str):
+        management_helper.add_excluded_backup_dir(server_id, dir_to_add)
+
+    @staticmethod
+    def del_excluded_backup_dir(server_id: int, dir_to_del: str):
+        management_helper.del_excluded_backup_dir(server_id, dir_to_del)

app/classes/minecraft/bedrock_ping.py

@@ -0,0 +1,107 @@
+import os
+import socket
+import time
+import psutil
+
+class BedrockPing:
+    magic = b'\x00\xff\xff\x00\xfe\xfe\xfe\xfe\xfd\xfd\xfd\xfd\x12\x34\x56\x78'
+    fields = { # (len, signed)
+        "byte": (1, False),
+        "long": (8, True),
+        "ulong": (8, False),
+        "magic": (16, False),
+        "short": (2, True),
+        "ushort": (2, False), #unsigned short
+        "string": (2, False), #strlen is ushort
+        "bool": (1, False),
+        "address": (7, False),
+        "uint24le": (3, False)
+    }
+    byte_order = 'big'
+
+    def __init__(self, bedrock_addr, bedrock_port, client_guid=0, timeout=5):
+        self.addr = bedrock_addr
+        self.port = bedrock_port
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.sock.settimeout(timeout)
+        self.proc = psutil.Process(os.getpid())
+        self.guid = client_guid
+        self.guid_bytes = self.guid.to_bytes(8, BedrockPing.byte_order)
+
+    @staticmethod
+    def __byter(in_val, to_type):
+        f = BedrockPing.fields[to_type]
+        return in_val.to_bytes(f[0], BedrockPing.byte_order, signed=f[1])
+
+    @staticmethod
+    def __slice(in_bytes, pattern):
+        ret = []
+        bi = 0 # bytes index
+        pi = 0 # pattern index
+        while bi < len(in_bytes):
+            try:
+                f = BedrockPing.fields[pattern[pi]]
+            except IndexError as index_error:
+                raise IndexError("Ran out of pattern with additional bytes remaining") from index_error
+            if pattern[pi] == "string":
+                shl = f[0] # string header length
+                sl = int.from_bytes(in_bytes[bi:bi+shl], BedrockPing.byte_order, signed=f[1]) # string length
+                l = shl+sl
+                ret.append(in_bytes[bi+shl:bi+shl+sl].decode('ascii'))
+            elif pattern[pi] == "magic":
+                l = f[0] # length of field
+                ret.append(in_bytes[bi:bi+l])
+            else:
+                l = f[0] # length of field
+                ret.append(int.from_bytes(in_bytes[bi:bi+l], BedrockPing.byte_order, signed=f[1]))
+            bi+=l
+            pi+=1
+        return ret
+
+    @staticmethod
+    def __get_time():
+        #return time.time_ns() // 1000000
+        return time.perf_counter_ns() // 1000000
+
+    def __sendping(self):
+        pack_id = BedrockPing.__byter(0x01, 'byte')
+        now = BedrockPing.__byter(BedrockPing.__get_time(), 'ulong')
+        guid = self.guid_bytes
+        d2s = pack_id+now+BedrockPing.magic+guid
+        #print("S:", d2s)
+        self.sock.sendto(d2s, (self.addr, self.port))
+
+    def __recvpong(self):
+        data = self.sock.recv(4096)
+        if data[0] == 0x1c:
+            ret = {}
+            sliced = BedrockPing.__slice(data,["byte","ulong","ulong","magic","string"])
+            if sliced[3] != BedrockPing.magic:
+                raise ValueError(f"Incorrect magic received ({sliced[3]})")
+            ret["server_guid"] = sliced[2]
+            ret["server_string_raw"] = sliced[4]
+            server_info = sliced[4].split(';')
+            ret["server_edition"] = server_info[0]
+            ret["server_motd"] = (server_info[1], server_info[7])
+            ret["server_protocol_version"] = server_info[2]
+            ret["server_version_name"] = server_info[3]
+            ret["server_player_count"] = server_info[4]
+            ret["server_player_max"] = server_info[5]
+            ret["server_uuid"] = server_info[6]
+            ret["server_game_mode"] = server_info[8]
+            ret["server_game_mode_num"] = server_info[9]
+            ret["server_port_ipv4"] = server_info[10]
+            ret["server_port_ipv6"] = server_info[11]
+            return ret
+        else:
+            raise ValueError(f"Incorrect packet type ({data[0]} detected")
+
+    def ping(self, retries=3):
+        rtr = retries
+        while rtr > 0:
+            try:
+                self.__sendping()
+                return self.__recvpong()
+            except ValueError as e:
+                print(f"E: {e}, checking next packet.  Retries remaining: {rtr}/{retries}")
+            rtr -= 1

app/classes/minecraft/mc_ping.py

@@ -3,9 +3,13 @@ import socket
 import base64
 import json
 import os
+import re
 import logging.config
+import uuid
+import random
 
 from app.classes.shared.console import console
+from app.classes.minecraft.bedrock_ping import BedrockPing
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +60,11 @@ class Server:
                 self.description = self.description['text']
 
         self.icon = base64.b64decode(data.get('favicon', '')[22:])
-        self.players = Players(data['players']).report()
+        try:
+            self.players = Players(data['players']).report()
+        except KeyError:
+            logger.error("Error geting player information key error")
+            self.players = []
         self.version = data['version']['name']
         self.protocol = data['version']['protocol']
 
@@ -162,69 +170,24 @@ def ping(ip, port):
 
             data += chunk
         logger.debug(f"Server reports this data on ping: {data}")
-        return Server(json.loads(data))
+        try:
+            return Server(json.loads(data))
+        except KeyError:
+            return {}
     finally:
         sock.close()
 
 # For the rest of requests see wiki.vg/Protocol
 def ping_bedrock(ip, port):
-    def read_var_int():
-        i = 0
-        j = 0
-        while True:
-            try:
-                k = sock.recvfrom(1024)
-            except:
-                return False
-            if not k:
-                return 0
-            k = k[0]
-            i |= (k & 0x7f) << (j * 7)
-            j += 1
-            if j > 5:
-                raise ValueError('var_int too big')
-            if not k & 0x80:
-                return i
-
-    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    sock.settimeout(2)
+    rd = random.Random()
     try:
-        sock.connect((ip, port))
-
+        #pylint: disable=consider-using-f-string
+        rd.seed(''.join(re.findall('..', '%012x' % uuid.getnode())))
+        client_guid = uuid.UUID(int=rd.getrandbits(32)).int
     except:
-        print("in first except")
-        return False
-
+        client_guid = 0
     try:
-        host = ip.encode('utf-8')
-        data = b''  # wiki.vg/Server_List_Ping
-        data += b'\x00'  # packet ID
-        data += b'\x04'  # protocol variant
-        data += struct.pack('>b', len(host)) + host
-        data += struct.pack('>H', port)
-        data += b'\x01'  # next state
-        data = struct.pack('>b', len(data)) + data
-        sock.sendall(data + b'\x01\x00')  # handshake + status ping
-        length = read_var_int()  # full packet length
-        if length < 10:
-            if length < 0:
-                return False
-            else:
-                return False
-        try:
-            sock.recvfrom(1024)  # packet type, 0 for pings
-        except:
-            return False
-        length = read_var_int()  # string length
-        data = b''
-        while len(data) != length:
-            print("in while")
-            chunk = sock.recv(length - len(data))
-            if not chunk:
-                return False
-
-            data += chunk
-        logger.debug(f"Server reports this data on ping: {data}")
-        return Server(json.loads(data))
-    finally:
-        sock.close()
+        brp = BedrockPing(ip, port, client_guid)
+        return brp.ping()
+    except:
+        logger.debug("Unable to get RakNet stats")

app/classes/minecraft/stats.py

@@ -1,4 +1,3 @@
-import os
 import json
 import logging
 import datetime
@@ -111,23 +110,11 @@ class Stats:
         return disk_data
 
     @staticmethod
-    def get_world_size(world_path):
+    def get_world_size(server_path):
 
         total_size = 0
 
-        # do a scan of the directories in the server path.
-        for root, dirs, _files in os.walk(world_path, topdown=False):
-
-            # for each directory we find
-            for name in dirs:
-
-                # if the directory name is "region"
-                if str(name) == "region":
-                    # log it!
-                    logger.debug("Path %s is called region. Getting directory size", os.path.join(root, name))
-
-                    # get this directory size, and add it to the total we have running.
-                    total_size += helper.get_dir_size(os.path.join(root, name))
+        total_size = helper.get_dir_size(server_path)
 
         level_total_size = helper.human_readable_file_size(total_size)
 
@@ -159,6 +146,26 @@ class Stats:
             'server_icon': server_icon
         }
 
+        return ping_data
+
+    @staticmethod
+    def parse_server_RakNet_ping(ping_obj: object):
+
+        try:
+            server_icon = base64.encodebytes(ping_obj['icon'])
+        except  Exception as e:
+            server_icon = False
+            logger.info(f"Unable to read the server icon : {e}")
+        ping_data = {
+            'online': ping_obj['server_player_count'],
+            'max': ping_obj['server_player_max'],
+            'players': [],
+            'server_description': ping_obj['server_edition'],
+            'server_version': ping_obj['server_version_name'],
+            'server_icon': server_icon
+        }
+
+
         return ping_data
 
     def get_server_players(self, server_id):
@@ -177,17 +184,16 @@ class Stats:
         server_port = server['server_port']
 
         logger.debug("Pinging {internal_ip} on port {server_port}")
-        if servers_helper.get_server_type_by_id(server_id) == 'minecraft-bedrock':
-            int_mc_ping = ping_bedrock(internal_ip, int(server_port))
-        else:
+        if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
             int_mc_ping = ping(internal_ip, int(server_port))
 
-        ping_data = {}
 
-        # if we got a good ping return, let's parse it
-        if int_mc_ping:
-            ping_data = self.parse_server_ping(int_mc_ping)
-            return ping_data['players']
+            ping_data = {}
+
+            # if we got a good ping return, let's parse it
+            if int_mc_ping:
+                ping_data = self.parse_server_ping(int_mc_ping)
+                return ping_data['players']
         return []
 
     def get_servers_stats(self):
@@ -210,12 +216,10 @@ class Stats:
             # get our server object, settings and data dictionaries
             server_obj = s.get('server_obj', None)
             server_obj.reload_server_settings()
-            server_settings = s.get('server_settings', {})
-            server_data = self.controller.get_server_data(server_id)
 
             # world data
-            world_name = server_settings.get('level-name', 'Unknown')
-            world_path = os.path.join(server_data.get('path', None), world_name)
+            server_name = server['server_name']
+            server_path = server['path']
 
             # process stats
             p_stats = self._get_process_stats(server_obj.process)
@@ -237,25 +241,47 @@ class Stats:
             # if we got a good ping return, let's parse it
             if int_mc_ping:
                 int_data = True
-                ping_data = self.parse_server_ping(int_mc_ping)
-
-            server_stats = {
-                'id': server_id,
-                'started': server_obj.get_start_time(),
-                'running': server_obj.check_running(),
-                'cpu': p_stats.get('cpu_usage', 0),
-                'mem': p_stats.get('memory_usage', 0),
-                "mem_percent": p_stats.get('mem_percentage', 0),
-                'world_name': world_name,
-                'world_size': self.get_world_size(world_path),
-                'server_port': server_port,
-                'int_ping_results': int_data,
-                'online': ping_data.get("online", False),
-                "max": ping_data.get("max", False),
-                'players': ping_data.get("players", False),
-                'desc': ping_data.get("server_description", False),
-                'version': ping_data.get("server_version", False)
-            }
+                if servers_helper.get_server_type_by_id(s['server_id']) == 'minecraft-bedrock':
+                    ping_data = self.parse_server_RakNet_ping(int_mc_ping)
+                else:
+                    ping_data = self.parse_server_ping(int_mc_ping)
+        #Makes sure we only show stats when a server is online otherwise people have gotten confused.
+            if server_obj.check_running():
+                server_stats = {
+                    'id': server_id,
+                    'started': server_obj.get_start_time(),
+                    'running': server_obj.check_running(),
+                    'cpu': p_stats.get('cpu_usage', 0),
+                    'mem': p_stats.get('memory_usage', 0),
+                    "mem_percent": p_stats.get('mem_percentage', 0),
+                    'world_name': server_name,
+                    'world_size': self.get_world_size(server_path),
+                    'server_port': server_port,
+                    'int_ping_results': int_data,
+                    'online': ping_data.get("online", False),
+                    "max": ping_data.get("max", False),
+                    'players': ping_data.get("players", False),
+                    'desc': ping_data.get("server_description", False),
+                    'version': ping_data.get("server_version", False)
+                }
+            else:
+                server_stats = {
+                    'id': server_id,
+                    'started': server_obj.get_start_time(),
+                    'running': server_obj.check_running(),
+                    'cpu': p_stats.get('cpu_usage', 0),
+                    'mem': p_stats.get('memory_usage', 0),
+                    "mem_percent": p_stats.get('mem_percentage', 0),
+                    'world_name': server_name,
+                    'world_size': self.get_world_size(server_path),
+                    'server_port': server_port,
+                    'int_ping_results': int_data,
+                    'online': False,
+                    "max": False,
+                    'players': False,
+                    'desc': False,
+                    'version': False
+                }
 
             # add this servers data to the stack
             server_stats_list.append(server_stats)
@@ -264,8 +290,30 @@ class Stats:
 
     def get_raw_server_stats(self, server_id):
 
+        try:
+            self.controller.get_server_obj(server_id)
+        except:
+            return {    'id': server_id,
+                        'started': False,
+                        'running': False,
+                        'cpu': 0,
+                        'mem': 0,
+                        "mem_percent": 0,
+                        'world_name': None,
+                        'world_size': None,
+                        'server_port': None,
+                        'int_ping_results': False,
+                        'online': False,
+                        'max': False,
+                        'players': False,
+                        'desc': False,
+                        'version': False,
+                        'icon': False}
+
         server_stats = {}
         server = self.controller.get_server_obj(server_id)
+        if not server:
+            return {}
         server_dt = servers_helper.get_server_data_by_id(server_id)
 
 
@@ -274,12 +322,10 @@ class Stats:
         # get our server object, settings and data dictionaries
         server_obj = self.controller.get_server_obj(server_id)
         server_obj.reload_server_settings()
-        server_settings = self.controller.get_server_settings(server_id)
-        server_data = self.controller.get_server_data(server_id)
 
         # world data
-        world_name = server_settings.get('level-name', 'Unknown')
-        world_path = os.path.join(server_data.get('path', None), world_name)
+        server_name = server_dt['server_name']
+        server_path = server_dt['path']
 
         # process stats
         p_stats = self._get_process_stats(server_obj.process)
@@ -299,30 +345,98 @@ class Stats:
 
         int_data = False
         ping_data = {}
+        #Makes sure we only show stats when a server is online otherwise people have gotten confused.
+        if server_obj.check_running():
+            # if we got a good ping return, let's parse it
+            if servers_helper.get_server_type_by_id(server_id) != 'minecraft-bedrock':
+                if int_mc_ping:
+                    int_data = True
+                    ping_data = self.parse_server_ping(int_mc_ping)
 
-        # if we got a good ping return, let's parse it
-        if int_mc_ping:
-            int_data = True
-            ping_data = self.parse_server_ping(int_mc_ping)
+                server_stats = {
+                    'id': server_id,
+                    'started': server_obj.get_start_time(),
+                    'running': server_obj.check_running(),
+                    'cpu': p_stats.get('cpu_usage', 0),
+                    'mem': p_stats.get('memory_usage', 0),
+                    "mem_percent": p_stats.get('mem_percentage', 0),
+                    'world_name': server_name,
+                    'world_size': self.get_world_size(server_path),
+                    'server_port': server_port,
+                    'int_ping_results': int_data,
+                    'online': ping_data.get("online", False),
+                    "max": ping_data.get("max", False),
+                    'players': ping_data.get("players", False),
+                    'desc': ping_data.get("server_description", False),
+                    'version': ping_data.get("server_version", False),
+                    'icon': ping_data.get("server_icon", False)
+                }
 
-        server_stats = {
-            'id': server_id,
-            'started': server_obj.get_start_time(),
-            'running': server_obj.check_running(),
-            'cpu': p_stats.get('cpu_usage', 0),
-            'mem': p_stats.get('memory_usage', 0),
-            "mem_percent": p_stats.get('mem_percentage', 0),
-            'world_name': world_name,
-            'world_size': self.get_world_size(world_path),
-            'server_port': server_port,
-            'int_ping_results': int_data,
-            'online': ping_data.get("online", False),
-            "max": ping_data.get("max", False),
-            'players': ping_data.get("players", False),
-            'desc': ping_data.get("server_description", False),
-            'version': ping_data.get("server_version", False),
-            'icon': ping_data.get("server_icon", False)
-        }
+            else:
+                if int_mc_ping:
+                    int_data = True
+                    ping_data = self.parse_server_RakNet_ping(int_mc_ping)
+                    try:
+                        server_icon = base64.encodebytes(ping_data['icon'])
+                    except  Exception as e:
+                        server_icon = False
+                        logger.info(f"Unable to read the server icon : {e}")
+
+                    server_stats = {
+                        'id': server_id,
+                        'started': server_obj.get_start_time(),
+                        'running': server_obj.check_running(),
+                        'cpu': p_stats.get('cpu_usage', 0),
+                        'mem': p_stats.get('memory_usage', 0),
+                        "mem_percent": p_stats.get('mem_percentage', 0),
+                        'world_name': server_name,
+                        'world_size': self.get_world_size(server_path),
+                        'server_port': server_port,
+                        'int_ping_results': int_data,
+                        'online': ping_data['online'],
+                        'max': ping_data['max'],
+                        'players': [],
+                        'desc': ping_data['server_description'],
+                        'version': ping_data['server_version'],
+                        'icon': server_icon
+                    }
+                else:
+                    server_stats = {
+                        'id': server_id,
+                        'started': server_obj.get_start_time(),
+                        'running': server_obj.check_running(),
+                        'cpu': p_stats.get('cpu_usage', 0),
+                        'mem': p_stats.get('memory_usage', 0),
+                        "mem_percent": p_stats.get('mem_percentage', 0),
+                        'world_name': server_name,
+                        'world_size': self.get_world_size(server_path),
+                        'server_port': server_port,
+                        'int_ping_results': int_data,
+                        'online': False,
+                        'max': False,
+                        'players': False,
+                        'desc': False,
+                        'version': False,
+                        'icon': False
+                    }
+        else:
+            server_stats = {
+                'id': server_id,
+                'started': server_obj.get_start_time(),
+                'running': server_obj.check_running(),
+                'cpu': p_stats.get('cpu_usage', 0),
+                'mem': p_stats.get('memory_usage', 0),
+                "mem_percent": p_stats.get('mem_percentage', 0),
+                'world_name': server_name,
+                'world_size': self.get_world_size(server_path),
+                'server_port': server_port,
+                'int_ping_results': int_data,
+                'online': False,
+                "max": False,
+                'players': False,
+                'desc': False,
+                'version': False
+            }
 
         return server_stats
 

app/classes/models/management.py

@@ -125,9 +125,10 @@ class Schedules(Model):
 #                                   Backups Class
 #************************************************************************************************
 class Backups(Model):
-    directories = CharField(null=True)
+    excluded_dirs = CharField(null=True)
     max_backups = IntegerField()
     server_id = ForeignKeyField(Servers, backref='backups_server')
+    compress = BooleanField(default=False)
     class Meta:
         table_name = 'backups'
         database = database
@@ -311,34 +312,41 @@ class helpers_management:
             row = Backups.select().where(Backups.server_id == server_id).join(Servers)[0]
             conf = {
                 "backup_path": row.server_id.backup_path,
-                "directories": row.directories,
+                "excluded_dirs": row.excluded_dirs,
                 "max_backups": row.max_backups,
-                "server_id": row.server_id.server_id
+                "server_id": row.server_id.server_id,
+                "compress": row.compress
             }
         except IndexError:
             conf = {
                 "backup_path": None,
-                "directories": None,
+                "excluded_dirs": None,
                 "max_backups": 0,
-                "server_id": server_id
+                "server_id": server_id,
+                "compress": False,
             }
         return conf
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None):
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False):
         logger.debug(f"Updating server {server_id} backup config with {locals()}")
         if Backups.select().where(Backups.server_id == server_id).count() != 0:
             new_row = False
             conf = {}
         else:
             conf = {
-                "directories": None,
+                "excluded_dirs": None,
                 "max_backups": 0,
-                "server_id":   server_id
+                "server_id":   server_id,
+                "compress": False
             }
             new_row = True
         if max_backups is not None:
             conf['max_backups'] = max_backups
+        if excluded_dirs is not None:
+            dirs_to_exclude = ",".join(excluded_dirs)
+            conf['excluded_dirs'] = dirs_to_exclude
+        conf['compress'] = compress
         if not new_row:
             with database.atomic():
                 if backup_path is not None:
@@ -355,5 +363,31 @@ class helpers_management:
                 Backups.create(**conf)
             logger.debug("Creating new backup record.")
 
+    def get_excluded_backup_dirs(self, server_id: int):
+        excluded_dirs = self.get_backup_config(server_id)['excluded_dirs']
+        if excluded_dirs is not None and excluded_dirs != "":
+            dir_list = excluded_dirs.split(",")
+        else:
+            dir_list = []
+        return dir_list
+
+    def add_excluded_backup_dir(self, server_id: int, dir_to_add: str):
+        dir_list = self.get_excluded_backup_dirs()
+        if dir_to_add not in dir_list:
+            dir_list.append(dir_to_add)
+            excluded_dirs = ",".join(dir_list)
+            self.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs)
+        else:
+            logger.debug(f"Not adding {dir_to_add} to excluded directories - already in the excluded directory list for server ID {server_id}")
+
+    def del_excluded_backup_dir(self, server_id: int, dir_to_del: str):
+        dir_list = self.get_excluded_backup_dirs()
+        if dir_to_del in dir_list:
+            dir_list.remove(dir_to_del)
+            excluded_dirs = ",".join(dir_list)
+            self.set_backup_config(server_id=server_id, excluded_dirs=excluded_dirs)
+        else:
+            logger.debug(f"Not removing {dir_to_del} from excluded directories - not in the excluded directory list for server ID {server_id}")
+
 
 management_helper = helpers_management()

app/classes/models/servers.py

@@ -157,10 +157,12 @@ class helper_servers:
     def get_all_servers_stats():
         servers = servers_helper.get_all_defined_servers()
         server_data = []
-
-        for s in servers:
-            latest = Server_Stats.select().where(Server_Stats.server_id == s.get('server_id')).order_by(Server_Stats.created.desc()).limit(1)
-            server_data.append({'server_data': s, "stats": db_helper.return_rows(latest)[0], "user_command_permission":True})
+        try:
+            for s in servers:
+                latest = Server_Stats.select().where(Server_Stats.server_id == s.get('server_id')).order_by(Server_Stats.created.desc()).limit(1)
+                server_data.append({'server_data': s, "stats": db_helper.return_rows(latest)[0], "user_command_permission":True})
+        except IndexError as ex:
+            logger.error(f"Stats collection failed with error: {ex}. Was a server just created?")
         return server_data
 
     @staticmethod

app/classes/shared/file_helpers.py

@@ -0,0 +1,111 @@
+import os
+import shutil
+import sys
+import logging
+import pathlib
+
+from app.classes.shared.console import console
+
+logger = logging.getLogger(__name__)
+
+try:
+    from zipfile import ZipFile, ZIP_DEFLATED
+
+except ModuleNotFoundError as err:
+    logger.critical(f"Import Error: Unable to load {err.name} module", exc_info=True)
+    console.critical(f"Import Error: Unable to load {err.name} module")
+    sys.exit(1)
+
+class FileHelpers:
+    allowed_quotes = [
+        "\"",
+        "'",
+        "`"
+     ]
+
+    def del_dirs(self, path):
+        path = pathlib.Path(path)
+        for sub in path.iterdir():
+            if sub.is_dir():
+                # Delete folder if it is a folder
+                self.del_dirs(sub)
+            else:
+                # Delete file if it is a file:
+                sub.unlink()
+
+        # This removes the top-level folder:
+        path.rmdir()
+        return True
+
+    @staticmethod
+    def del_file(path):
+        path = pathlib.Path(path)
+        try:
+            logger.debug(f"Deleting file: {path}")
+            #Remove the file
+            os.remove(path)
+            return True
+        except FileNotFoundError:
+            logger.error(f"Path specified is not a file or does not exist. {path}")
+            return False
+
+    @staticmethod
+    def copy_dir(src_path, dest_path, dirs_exist_ok=False):
+        # pylint: disable=unexpected-keyword-arg
+        shutil.copytree(src_path, dest_path, dirs_exist_ok=dirs_exist_ok)
+
+    @staticmethod
+    def copy_file(src_path, dest_path):
+        shutil.copy(src_path, dest_path)
+
+    def move_dir(self, src_path, dest_path):
+        self.copy_dir(src_path, dest_path)
+        self.del_dirs(src_path)
+
+    def move_file(self, src_path, dest_path):
+        self.copy_file(src_path, dest_path)
+        self.del_file(src_path)
+
+    @staticmethod
+    def make_archive(path_to_destination, path_to_zip):
+        # create a ZipFile object
+        path_to_destination += '.zip'
+        with ZipFile(path_to_destination, 'w') as z:
+            for root, _dirs, files in os.walk(path_to_zip, topdown=True):
+                ziproot = path_to_zip
+                for file in files:
+                    try:
+                        logger.info(f"backing up: {os.path.join(root, file)}")
+                        if os.name == "nt":
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, ""), file))
+                        else:
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, "/"), file))
+
+                    except Exception as e:
+                        logger.warning(f"Error backing up: {os.path.join(root, file)}! - Error was: {e}")
+
+
+        return True
+
+    @staticmethod
+    def make_compressed_archive(path_to_destination, path_to_zip):
+        # create a ZipFile object
+        path_to_destination += '.zip'
+        with ZipFile(path_to_destination, 'w', ZIP_DEFLATED) as z:
+            for root, _dirs, files in os.walk(path_to_zip, topdown=True):
+                ziproot = path_to_zip
+                for file in files:
+                    try:
+                        logger.info(f"backing up: {os.path.join(root, file)}")
+                        if os.name == "nt":
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, ""), file))
+                        else:
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, "/"), file))
+
+                    except Exception as e:
+                        logger.warning(f"Error backing up: {os.path.join(root, file)}! - Error was: {e}")
+
+
+        return True
+
+file_helper = FileHelpers()

app/classes/shared/helpers.py

@@ -13,15 +13,16 @@ import logging
 import html
 import zipfile
 import pathlib
-import shutil
 import ctypes
 from datetime import datetime
 from socket import gethostname
 from contextlib import suppress
+import psutil
 from requests import get
 
 from app.classes.web.websocket_helper import websocket_helper
 from app.classes.shared.console import console
+from app.classes.shared.file_helpers import file_helper
 
 logger = logging.getLogger(__name__)
 
@@ -369,7 +370,7 @@ class Helpers:
 
                 for item in os.listdir(full_root_path):
                     try:
-                        shutil.move(os.path.join(full_root_path, item), os.path.join(new_dir, item))
+                        file_helper.move_dir(os.path.join(full_root_path, item), os.path.join(new_dir, item))
                     except Exception as ex:
                         logger.error(f'ERROR IN ZIP IMPORT: {ex}')
             except Exception as ex:
@@ -481,11 +482,18 @@ class Helpers:
                 pid = data.get('pid')
                 started = data.get('started')
                 console.critical(f"Another Crafty Controller agent seems to be running...\npid: {pid} \nstarted on: {started}")
+                if psutil.pid_exists(pid):
+                    logger.critical("Found running crafty process. Exiting.")
+                    sys.exit(1)
+                else:
+                    logger.info("No process found for pid. Assuming crafty crashed. Deleting stale session.lock")
+                    os.remove(self.session_file)
+
             except Exception as e:
                 logger.error(f"Failed to locate existing session.lock with error: {e} ")
                 console.error(f"Failed to locate existing session.lock with error: {e} ")
 
-            sys.exit(1)
+                sys.exit(1)
 
         pid = os.getpid()
         now = datetime.now()
@@ -652,8 +660,15 @@ class Helpers:
 
     @staticmethod
     def generate_tree(folder, output=""):
+        dir_list = []
+        unsorted_files = []
         file_list = os.listdir(folder)
-        file_list = sorted(file_list, key=str.casefold)
+        for item in file_list:
+            if os.path.isdir(os.path.join(folder, item)):
+                dir_list.append(item)
+            else:
+                unsorted_files.append(item)
+        file_list = sorted(dir_list, key=str.casefold) + sorted(unsorted_files, key=str.casefold)
         for raw_filename in file_list:
             filename = html.escape(raw_filename)
             rel = os.path.join(folder, raw_filename)
@@ -673,7 +688,7 @@ class Helpers:
             else:
                 if filename != "crafty_managed.txt":
                     output += f"""<li
-                    class="tree-item tree-ctx-item tree-file"
+                    class="tree-nested d-block tree-ctx-item tree-file"
                     data-path="{dpath}"
                     data-name="{filename}"
                     onclick="clickOnFile(event)"><span style="margin-right: 6px;"><i class="far fa-file"></i></span>{filename}</li>"""
@@ -681,8 +696,15 @@ class Helpers:
 
     @staticmethod
     def generate_dir(folder, output=""):
+        dir_list = []
+        unsorted_files = []
         file_list = os.listdir(folder)
-        file_list = sorted(file_list, key=str.casefold)
+        for item in file_list:
+            if os.path.isdir(os.path.join(folder, item)):
+                dir_list.append(item)
+            else:
+                unsorted_files.append(item)
+        file_list = sorted(dir_list, key=str.casefold) + sorted(unsorted_files, key=str.casefold)
         output += \
     f"""<ul class="tree-nested d-block" id="{folder}ul">"""\
 
@@ -704,7 +726,7 @@ class Helpers:
             else:
                 if filename != "crafty_managed.txt":
                     output += f"""<li
-                    class="tree-item tree-ctx-item tree-file"
+                    class="tree-nested d-block tree-ctx-item tree-file"
                     data-path="{dpath}"
                     data-name="{filename}"
                     onclick="clickOnFile(event)"><span style="margin-right: 6px;"><i class="far fa-file"></i></span>{filename}</li>"""
@@ -773,6 +795,12 @@ class Helpers:
                 websocket_helper.broadcast_user(user_id, 'send_temp_path',{
                 'path': tempDir
                 })
+    @staticmethod
+    def backup_select(path, user_id):
+        if user_id:
+            websocket_helper.broadcast_user(user_id, 'send_temp_path',{
+            'path': path
+        })
 
     @staticmethod
     def unzip_backup_archive(backup_path, zip_name):
@@ -804,7 +832,7 @@ class Helpers:
     @staticmethod
     def copy_files(source, dest):
         if os.path.isfile(source):
-            shutil.copyfile(source, dest)
+            file_helper.copy_file(source, dest)
             logger.info("Copying jar %s to %s", source, dest)
         else:
             logger.info("Source jar does not exist.")

app/classes/shared/main_controller.py

@@ -1,10 +1,9 @@
 import os
 import pathlib
+import shutil
 import time
 import logging
-import shutil
 import tempfile
-from distutils import dir_util
 from typing import Union
 from peewee import DoesNotExist
 
@@ -23,6 +22,7 @@ from app.classes.models.servers import servers_helper
 from app.classes.shared.console import console
 from app.classes.shared.helpers import helper
 from app.classes.shared.server import Server
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.minecraft.server_props import ServerProps
 from app.classes.minecraft.serverjars import server_jar_obj
@@ -153,13 +153,13 @@ class Controller:
             final_path = os.path.join(server_path, str(server['server_name']))
             os.mkdir(final_path)
             try:
-                shutil.copy(server['log_path'], final_path)
+                file_helper.copy_file(server['log_path'], final_path)
             except Exception as e:
                 logger.warning(f"Failed to copy file with error: {e}")
         #Copy crafty logs to archive dir
         full_log_name = os.path.join(crafty_path, 'logs')
-        shutil.copytree(os.path.join(self.project_root, 'logs'), full_log_name)
-        shutil.make_archive(tempZipStorage, "zip", tempDir)
+        file_helper.copy_dir(os.path.join(self.project_root, 'logs'), full_log_name)
+        file_helper.make_archive(tempZipStorage, tempDir)
 
         tempZipStorage += '.zip'
         websocket_helper.broadcast_user(exec_user['user_id'], 'send_logs_bootbox', {
@@ -329,7 +329,10 @@ class Controller:
         helper.ensure_dir_exists(new_server_dir)
         helper.ensure_dir_exists(backup_path)
         server_path = helper.get_os_understandable_path(server_path)
-        dir_util.copy_tree(server_path, new_server_dir)
+        try:
+            file_helper.copy_dir(server_path, new_server_dir, True)
+        except shutil.Error as ex:
+            logger.error(f"Server import failed with error: {ex}")
 
         has_properties = False
         for item in os.listdir(new_server_dir):
@@ -374,7 +377,10 @@ class Controller:
             if str(item) == 'server.properties':
                 has_properties = True
             try:
-                shutil.move(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                if not os.path.isdir(os.path.join(tempDir, item)):
+                    file_helper.move_file(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                else:
+                    file_helper.move_dir(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
             except Exception as ex:
                 logger.error(f'ERROR IN ZIP IMPORT: {ex}')
         if not has_properties:
@@ -415,7 +421,10 @@ class Controller:
         helper.ensure_dir_exists(new_server_dir)
         helper.ensure_dir_exists(backup_path)
         server_path = helper.get_os_understandable_path(server_path)
-        dir_util.copy_tree(server_path, new_server_dir)
+        try:
+            file_helper.copy_dir(server_path, new_server_dir, True)
+        except shutil.Error as ex:
+            logger.error(f"Server import failed with error: {ex}")
 
         has_properties = False
         for item in os.listdir(new_server_dir):
@@ -440,7 +449,9 @@ class Controller:
 
         new_id = self.register_server(server_name, server_id, new_server_dir, backup_path, server_command, server_exe,
                                       server_log_file, server_stop, port, server_type='minecraft-bedrock')
-        os.chmod(full_jar_path, 0o2775)
+        if os.name != "nt":
+            if helper.check_file_exists(full_jar_path):
+                os.chmod(full_jar_path, 0o2775)
         return new_id
 
     def import_bedrock_zip_server(self, server_name: str, zip_path: str, server_exe: str, port: int):
@@ -462,7 +473,10 @@ class Controller:
             if str(item) == 'server.properties':
                 has_properties = True
             try:
-                shutil.move(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                if not os.path.isdir(os.path.join(tempDir, item)):
+                    file_helper.move_file(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
+                else:
+                    file_helper.move_dir(os.path.join(tempDir, item), os.path.join(new_server_dir, item))
             except Exception as ex:
                 logger.error(f'ERROR IN ZIP IMPORT: {ex}')
         if not has_properties:
@@ -484,7 +498,10 @@ class Controller:
 
         new_id = self.register_server(server_name, server_id, new_server_dir, backup_path, server_command, server_exe,
                                       server_log_file, server_stop, port, server_type='minecraft-bedrock')
-        os.chmod(full_jar_path, 0o2775)
+        if os.name != "nt":
+            if helper.check_file_exists(full_jar_path):
+                os.chmod(full_jar_path, 0o2775)
+
         return new_id
 
     #************************************************************************************************
@@ -558,11 +575,11 @@ class Controller:
                     self.stop_server(server_id)
                 if files:
                     try:
-                        shutil.rmtree(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['path']))
+                        file_helper.del_dirs(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['path']))
                     except Exception as e:
                         logger.error(f"Unable to delete server files for server with ID: {server_id} with error logged: {e}")
                     if helper.check_path_exists(self.servers.get_server_data_by_id(server_id)['backup_path']):
-                        shutil.rmtree(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['backup_path']))
+                        file_helper.del_dirs(helper.get_os_understandable_path(self.servers.get_server_data_by_id(server_id)['backup_path']))
 
 
                 #Cleanup scheduled tasks

app/classes/shared/server.py

@@ -5,9 +5,9 @@ import time
 import datetime
 import threading
 import logging.config
-import shutil
 import subprocess
 import html
+import tempfile
 from apscheduler.schedulers.background import BackgroundScheduler
 #TZLocal is set as a hidden import on win pipeline
 from tzlocal import get_localzone
@@ -20,6 +20,7 @@ from app.classes.models.server_permissions import server_permissions
 from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
 from app.classes.shared.translation import translation
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.web.websocket_helper import websocket_helper
 
@@ -574,21 +575,57 @@ class Server:
 
     def a_backup_server(self):
         logger.info(f"Starting server {self.name} (ID {self.server_id}) backup")
+        server_users = server_permissions.get_server_user_list(self.server_id)
+        for user in server_users:
+            websocket_helper.broadcast_user(user, 'notification', translation.translate('notify',
+            'backupStarted', users_helper.get_user_lang_by_id(user)) + self.name)
+        time.sleep(3)
         self.is_backingup = True
         conf = management_helper.get_backup_config(self.server_id)
         try:
             backup_filename = f"{self.settings['backup_path']}/{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
             logger.info(f"Creating backup of server '{self.settings['server_name']}'" +
                         f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'")
-            shutil.make_archive(helper.get_os_understandable_path(backup_filename), 'zip', self.server_path)
+
+            tempDir = tempfile.mkdtemp()
+            # pylint: disable=unexpected-keyword-arg
+            file_helper.copy_dir(self.server_path, tempDir, dirs_exist_ok=True)
+            excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id)
+            server_dir = helper.get_os_understandable_path(self.settings['path'])
+
+            for my_dir in excluded_dirs:
+                # Take the full path of the excluded dir and replace the server path with the temp path
+                # This is so that we're only deleting excluded dirs from the temp path and not the server path
+                excluded_dir = helper.get_os_understandable_path(my_dir).replace(server_dir, helper.get_os_understandable_path(tempDir))
+                # Next, check to see if it is a directory
+                if os.path.isdir(excluded_dir):
+                    # If it is a directory, recursively delete the entire directory from the backup
+                    file_helper.del_dirs(excluded_dir)
+                else:
+                    # If not, just remove the file
+                    os.remove(excluded_dir)
+            if conf['compress']:
+                logger.debug("Found compress backup to be true. Calling compressed archive")
+                file_helper.make_compressed_archive(helper.get_os_understandable_path(backup_filename), tempDir)
+            else:
+                logger.debug("Found compress backup to be false. Calling NON-compressed archive")
+                file_helper.make_archive(helper.get_os_understandable_path(backup_filename), tempDir)
+
             while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0:
                 backup_list = self.list_backups()
                 oldfile = backup_list[0]
                 oldfile_path = f"{conf['backup_path']}/{oldfile['path']}"
                 logger.info(f"Removing old backup '{oldfile['path']}'")
                 os.remove(helper.get_os_understandable_path(oldfile_path))
+
             self.is_backingup = False
+            file_helper.del_dirs(tempDir)
             logger.info(f"Backup of server: {self.name} completed")
+            server_users = server_permissions.get_server_user_list(self.server_id)
+            for user in server_users:
+                websocket_helper.broadcast_user(user, 'notification', translation.translate('notify', 'backupComplete',
+                users_helper.get_user_lang_by_id(user)) + self.name)
+            time.sleep(3)
             return
         except:
             logger.exception(f"Failed to create backup of server {self.name} (ID {self.server_id})")

app/classes/web/ajax_handler.py

@@ -2,12 +2,15 @@ import os
 import html
 import re
 import logging
+import time
 import tornado.web
 import tornado.escape
 import bleach
 
 from app.classes.shared.console import console
 from app.classes.shared.helpers import helper
+from app.classes.web.websocket_helper import websocket_helper
+from app.classes.shared.translation import translation
 from app.classes.shared.server import ServerOutBuf
 
 from app.classes.web.base_handler import BaseHandler
@@ -99,6 +102,155 @@ class AjaxHandler(BaseHandler):
                         helper.generate_zip_dir(path))
             self.finish()
 
+        elif page == "get_backup_tree":
+            server_id = self.get_argument('id', None)
+            folder = self.get_argument('path', None)
+
+            output = ""
+
+            dir_list = []
+            unsorted_files = []
+            file_list = os.listdir(folder)
+            for item in file_list:
+                if os.path.isdir(os.path.join(folder, item)):
+                    dir_list.append(item)
+                else:
+                    unsorted_files.append(item)
+            file_list = sorted(dir_list, key=str.casefold) + sorted(unsorted_files, key=str.casefold)
+            output += \
+        f"""<ul class="tree-nested d-block" id="{folder}ul">"""\
+
+            for raw_filename in file_list:
+                filename = html.escape(raw_filename)
+                rel = os.path.join(folder, raw_filename)
+                dpath = os.path.join(folder, filename)
+                if str(dpath) in self.controller.management.get_excluded_backup_dirs(server_id):
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" class="checkBoxClass" name="root_path" value="{dpath}" checked>
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>
+                            \n"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-nested d-block tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' class="checkBoxClass" name='root_path' value="{dpath}" checked><span style="margin-right: 6px;">
+                        <i class="far fa-file"></i></span></input>{filename}</li>"""
+
+                else:
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" class="checkBoxClass" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>
+                            \n"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-nested d-block tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' class="checkBoxClass" name='root_path' value="{dpath}">
+                        <span style="margin-right: 6px;"><i class="far fa-file"></i></span></input>{filename}</li>"""
+            self.write(helper.get_os_understandable_path(folder) + '\n' +
+                        output)
+            self.finish()
+
+        elif page == "get_backup_dir":
+            server_id = self.get_argument('id', None)
+            folder = self.get_argument('path', None)
+            output = ""
+
+            dir_list = []
+            unsorted_files = []
+            file_list = os.listdir(folder)
+            for item in file_list:
+                if os.path.isdir(os.path.join(folder, item)):
+                    dir_list.append(item)
+                else:
+                    unsorted_files.append(item)
+            file_list = sorted(dir_list, key=str.casefold) + sorted(unsorted_files, key=str.casefold)
+            output += \
+        f"""<ul class="tree-nested d-block" id="{folder}ul">"""\
+
+            for raw_filename in file_list:
+                filename = html.escape(raw_filename)
+                rel = os.path.join(folder, raw_filename)
+                dpath = os.path.join(folder, filename)
+                if str(dpath) in self.controller.management.get_excluded_backup_dirs(server_id):
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' name='root_path' value='{dpath}'><span style="margin-right: 6px;">
+                        <i class="far fa-file"></i></span></input>{filename}</li>"""
+
+                else:
+                    if os.path.isdir(rel):
+                        output += \
+                            f"""<li class="tree-item" data-path="{dpath}">
+                            \n<div id="{dpath}" data-path="{dpath}" data-name="{filename}" class="tree-caret tree-ctx-item tree-folder">
+                            <input type="checkbox" name="root_path" value="{dpath}">
+                            <span id="{dpath}span" class="files-tree-title" data-path="{dpath}" data-name="{filename}" onclick="getDirView(event)">
+                            <i class="far fa-folder"></i>
+                            <i class="far fa-folder-open"></i>
+                            <strong>{filename}</strong>
+                            </span>
+                            </input></div><li>"""\
+
+                    else:
+                        output += f"""<li
+                        class="tree-item tree-ctx-item tree-file"
+                        data-path="{dpath}"
+                        data-name="{filename}"
+                        onclick=""><input type='checkbox' name='root_path' value='{dpath}'>
+                        <span style="margin-right: 6px;"><i class="far fa-file"></i></span></input>{filename}</li>"""
+
+            self.write(helper.get_os_understandable_path(folder) + '\n' +
+                        output)
+            self.finish()
+
+        elif page == "get_dir":
+            server_id = self.get_argument('id', None)
+            path = self.get_argument('path', None)
+
+            if not self.check_server_id(server_id, 'get_tree'):
+                return
+            else:
+                server_id = bleach.clean(server_id)
+
+            if helper.validate_traversal(self.controller.servers.get_server_data_by_id(server_id)['path'], path):
+                self.write(helper.get_os_understandable_path(path) + '\n' +
+                        helper.generate_dir(path))
+            self.finish()
 
     @tornado.web.authenticated
     def post(self, page):
@@ -196,7 +348,21 @@ class AjaxHandler(BaseHandler):
 
         elif page == "unzip_server":
             path = self.get_argument('path', None)
-            helper.unzipServer(path, exec_user['user_id'])
+            if helper.check_file_exists(path):
+                helper.unzipServer(path, exec_user['user_id'])
+            else:
+                user_id = exec_user['user_id']
+                if user_id:
+                    time.sleep(5)
+                    user_lang = self.controller.users.get_user_lang_by_id(user_id)
+                    websocket_helper.broadcast_user(user_id, 'send_start_error',{
+                    'error': translation.translate('error', 'no-file', user_lang)
+                })
+            return
+
+        elif page == "backup_select":
+            path = self.get_argument('path', None)
+            helper.backup_select(path, exec_user['user_id'])
             return
 
 

app/classes/web/file_handler.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 import logging
 import tornado.web
 import tornado.escape
@@ -7,6 +6,7 @@ import bleach
 
 from app.classes.shared.console import console
 from app.classes.shared.helpers import helper
+from app.classes.shared.file_helpers import file_helper
 
 from app.classes.web.base_handler import BaseHandler
 from app.classes.models.server_permissions import Enum_Permissions_Server
@@ -232,7 +232,7 @@ class FileHandler(BaseHandler):
                 return
 
             # Delete the file
-            os.remove(file_path)
+            file_helper.del_file(file_path)
 
         elif page == "del_dir":
             if not permissions['Files'] in user_perms:
@@ -258,7 +258,8 @@ class FileHandler(BaseHandler):
             # Delete the directory
             # os.rmdir(dir_path)     # Would only remove empty directories
             if helper.validate_traversal(helper.get_os_understandable_path(server_info['path']), dir_path):
-                shutil.rmtree(dir_path)  # Removes also when there are contents
+                # Removes also when there are contents
+                file_helper.del_dirs(dir_path)
 
     @tornado.web.authenticated
     def put(self, page):

app/classes/web/panel_handler.py

@@ -204,6 +204,29 @@ class PanelHandler(BaseHandler):
                 exec_user_role.add(role['role_name'])
             defined_servers = self.controller.servers.get_authorized_servers(exec_user["user_id"])
 
+
+        user_order = self.controller.users.get_user_by_id(exec_user['user_id'])
+        user_order = user_order['server_order'].split(',')
+        page_servers = []
+        server_ids = []
+
+        for server_id in user_order:
+            for server in defined_servers:
+                if str(server['server_id']) == str(server_id):
+                    page_servers.append(server)
+
+
+        for server in defined_servers:
+            server_ids.append(str(server['server_id']))
+            if server not in page_servers:
+                page_servers.append(server)
+        for server_id in user_order:
+            #remove IDs in list that user no longer has access to
+            if str(server_id) not in server_ids:
+                user_order.remove(server_id)
+        defined_servers = page_servers
+
+
         page_data: Dict[str, Any] = {
             # todo: make this actually pull and compare version data
             'update_available': False,
@@ -428,6 +451,15 @@ class PanelHandler(BaseHandler):
                         return
                 server_info = self.controller.servers.get_server_data_by_id(server_id)
                 page_data['backup_config'] = self.controller.management.get_backup_config(server_id)
+                exclusions = []
+                page_data['exclusions'] = self.controller.management.get_excluded_backup_dirs(server_id)
+                #makes it so relative path is the only thing shown
+                for file in page_data['exclusions']:
+                    if helper.is_os_windows():
+                        exclusions.append(file.replace(server_info['path']+'\\', ""))
+                    else:
+                        exclusions.append(file.replace(server_info['path']+'/', ""))
+                page_data['exclusions'] = exclusions
                 self.controller.refresh_server_settings(server_id)
                 try:
                     page_data['backup_list'] = server.list_backups()
@@ -574,10 +606,11 @@ class PanelHandler(BaseHandler):
                 page_data['super-disabled'] = ''
             else:
                 page_data['super-disabled'] = 'disabled'
-            for file in os.listdir(os.path.join(helper.root_dir, 'app', 'translations')):
+            for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))):
                 if file.endswith('.json'):
-                    if file != str(page_data['languages'][0] + '.json'):
-                        page_data['languages'].append(file.split('.')[0])
+                    if file not in helper.get_setting('disabled_language_files'):
+                        if file != str(page_data['languages'][0] + '.json'):
+                            page_data['languages'].append(file.split('.')[0])
 
             template = "panel/panel_edit_user.html"
 
@@ -706,8 +739,9 @@ class PanelHandler(BaseHandler):
 
             for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))):
                 if file.endswith('.json'):
-                    if file != str(page_data['languages'][0] + '.json'):
-                        page_data['languages'].append(file.split('.')[0])
+                    if file not in helper.get_setting('disabled_language_files'):
+                        if file != str(page_data['languages'][0] + '.json'):
+                            page_data['languages'].append(file.split('.')[0])
 
             if user_id is None:
                 self.redirect("/panel/error?error=Invalid User ID")
@@ -1027,6 +1061,12 @@ class PanelHandler(BaseHandler):
             logger.debug(self.request.arguments)
             server_id = self.get_argument('id', None)
             server_obj = self.controller.servers.get_server_obj(server_id)
+            compress = self.get_argument('compress', False)
+            check_changed = self.get_argument('changed')
+            if str(check_changed) == str(1):
+                checked = self.get_body_arguments('root_path')
+            else:
+                checked = self.controller.management.get_excluded_backup_dirs(server_id)
             if superuser:
                 backup_path = bleach.clean(self.get_argument('backup_path', None))
                 if helper.is_os_windows():
@@ -1052,7 +1092,7 @@ class PanelHandler(BaseHandler):
             server_obj = self.controller.servers.get_server_obj(server_id)
             server_obj.backup_path = backup_path
             self.controller.servers.update_server(server_obj)
-            self.controller.management.set_backup_config(server_id, max_backups=max_backups)
+            self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked, compress=bool(compress))
 
             self.controller.management.add_to_audit_log(exec_user['user_id'],
                                        f"Edited server {server_id}: updated backups",
@@ -1152,7 +1192,9 @@ class PanelHandler(BaseHandler):
                         "start_time": sch_time,
                         "enabled": enabled,
                         "one_time": one_time,
-                        "cron_string": ''
+                        "cron_string": '',
+                        "parent": None,
+                        "delay": 0
                     }
                 elif difficulty == "reaction":
                     job_data = {
@@ -1300,7 +1342,9 @@ class PanelHandler(BaseHandler):
                         "start_time": sch_time,
                         "enabled": enabled,
                         "one_time": one_time,
-                        "cron_string": ''
+                        "cron_string": '',
+                        "parent": None,
+                        "delay": 0
                     }
                 elif difficulty == "advanced":
                     job_data = {

app/classes/web/server_handler.py

@@ -2,12 +2,12 @@ import sys
 import json
 import logging
 import os
-import shutil
 import libgravatar
 import requests
 
 from app.classes.shared.helpers import helper
 from app.classes.shared.console import console
+from app.classes.shared.file_helpers import file_helper
 from app.classes.web.base_handler import BaseHandler
 from app.classes.models.crafty_permissions import Enum_Permissions_Crafty
 from app.classes.minecraft.serverjars import server_jar_obj
@@ -169,7 +169,7 @@ class ServerHandler(BaseHandler):
                     new_server_path = os.path.join(helper.servers_dir, new_server_uuid)
 
                     # copy the old server
-                    shutil.copytree(server_data.get('path'), new_server_path)
+                    file_helper.copy_dir(server_data.get('path'), new_server_path)
 
                     # TODO get old server DB data to individual variables
                     stop_command = server_data.get('stop_command')
@@ -177,7 +177,7 @@ class ServerHandler(BaseHandler):
                     new_executable = server_data.get('executable')
                     new_server_log_file = str(helper.get_os_understandable_path(server_data.get('log_path'))).replace(server_uuid, new_server_uuid)
                     server_port = server_data.get('server_port')
-                    server_type = server_data.get('server_type')
+                    server_type = server_data.get('type')
 
                     self.controller.servers.create_server(new_server_name,
                                                           new_server_uuid,
@@ -250,7 +250,7 @@ class ServerHandler(BaseHandler):
                                            new_server_id,
                                            self.get_remote_ip())
                 #deletes temp dir
-                shutil.rmtree(zip_path)
+                file_helper.del_dirs(zip_path)
             else:
                 if len(server_parts) != 2:
                     self.redirect("/panel/error?error=Invalid server data")
@@ -304,7 +304,7 @@ class ServerHandler(BaseHandler):
                 return
 
             if import_type == 'import_jar':
-                good_path = self.controller.verify_jar_server(import_server_path, import_server_jar)
+                good_path = self.controller.verify_jar_server(import_server_path, import_server_exe)
 
                 if not good_path:
                     self.redirect("/panel/error?error=Server path or Server Jar not found!")
@@ -333,7 +333,7 @@ class ServerHandler(BaseHandler):
                                            new_server_id,
                                            self.get_remote_ip())
                 #deletes temp dir
-                shutil.rmtree(zip_path)
+                file_helper.del_dirs(zip_path)
             else:
                 if len(server_parts) != 2:
                     self.redirect("/panel/error?error=Invalid server data")

app/config/config.json

@@ -14,6 +14,7 @@
   "virtual_terminal_lines": 70,
   "max_log_lines": 700,
   "max_audit_entries": 300,
+  "disabled_language_files": ["lol_EN.json", ""],
   "keywords": ["help", "chunk"],
   "allow_nsfw_profile_pictures": false
 }

app/frontend/templates/panel/dashboard.html

@@ -116,7 +116,7 @@
                   <th>{{ translate('dashboard', 'actions', data['lang']) }}</th>
                   <th>{{ translate('dashboard', 'cpuUsage', data['lang']) }}</th>
                   <th>{{ translate('dashboard', 'memUsage', data['lang']) }}</th>
-                  <th>{{ translate('dashboard', 'world', data['lang']) }}</th>
+                  <th>{{ translate('dashboard', 'size', data['lang']) }}</th>
                   <th>{{ translate('dashboard', 'players', data['lang']) }}</th>
                   <th>{{ translate('dashboard', 'status', data['lang']) }}</th>
                 </tr>
@@ -215,7 +215,7 @@
                     {% end %}
                   </td>
                   <td id="server_world_{{server['server_data']['server_id']}}">
-                    {{ server['stats']['world_name'] }} : {{ server['stats']['world_size'] }}
+                    {{ server['stats']['world_size'] }}
                   </td>
                   <td id="server_desc_{{server['server_data']['server_id']}}">
                     {% if server['stats']['int_ping_results'] %}
@@ -396,7 +396,7 @@
     server_mem.innerHTML = `<div class="progress mb-1" data-toggle="tooltip" data-placement="top" title="`+ server_mem +`"><div class="progress-bar `+ mem_status + `" role="progressbar" style="width: `+ server.mem_percent + `%" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"></div></div>`+ server.mem_percent +`% - ` + total_mem;
 
     /* Update World Infos */
-    server_world.innerHTML = server.world_name + ` : ` + server.world_size
+    server_world.innerHTML = server.world_size
 
     /* Update Server Infos */
     if (server.int_ping_results) {

app/frontend/templates/panel/panel_edit_user.html

@@ -239,13 +239,15 @@
 
 {% block js %}
 <script>
+  const userId = new URLSearchParams(document.location.search).get('id')
+
   $( ".delete-user" ).click(function() {
     var file_to_del = $(this).data("file");
 
-    console.log("User to delete is {{ data['user']['username']  }}");
+    console.log("User to delete is "+userId);
 
     bootbox.confirm({
-        title: "{% raw translate('userConfig', 'deleteUser', data['lang']) %}"+"{{ data['user']['username']  }}",
+        title: "{% raw translate('userConfig', 'deleteUser', data['lang']) %} "+userId,
         message: "{{ translate('userConfig', 'confirmDelete', data['lang']) }}",
         buttons: {
             cancel: {
@@ -259,7 +261,7 @@
         callback: function (result) {
             console.log(result);
             if (result == true) {
-              location.href="/panel/remove_user?id={{ data['user']['user_id'] }}";
+              location.href="/panel/remove_user?id="+userId;
             }
         }
     });

app/frontend/templates/panel/parts/details_stats.html

@@ -136,15 +136,8 @@
 
     /* TODO Update each element */
     if (server.running){
-      if (server['stats']['waiting_start']){
-        server_status.setAttribute("class", "text-warning");
-        server_status.innerHTML = `{{ translate('serverStats', 'starting', data['lang']) }}`;
-      }
-      else
-      {
         server_status.setAttribute("class", "text-success");
         server_status.innerHTML = `{{ translate('serverStats', 'online', data['lang']) }}`;
-      }
 
       startedUTC = server.started;
       startedUTC = moment.utc(startedUTC, 'YYYY-MM-DD HH:mm:ss');

app/frontend/templates/panel/parts/server_controls_list.html

@@ -36,7 +36,7 @@
         <i class="fas fa-cogs"></i>{{ translate('serverDetails', 'config', data['lang']) }}</a>
     </li>
     {% end %}
-    {% if data['permissions']['Players'] in data['user_permissions'] %}
+    {% if data['permissions']['Players'] in data['user_permissions'] and data['server_data']['type'] != 'minecraft-bedrock' %}
     <li class="nav-item term-nav-item">
         <a class="nav-link {% if data['active_link'] == 'admin_controls' %}active{% end %}" href="/panel/server_detail?id={{ data['server_stats']['server_id']['server_id'] }}&subpage=admin_controls" role="tab" aria-selected="true">
         <i class="fas fa-users"></i>{{ translate('serverDetails', 'playerControls', data['lang']) }}</a>

app/frontend/templates/panel/server_backup.html

@@ -35,6 +35,8 @@
 
             <div class="row">
                 <div class="col-md-6 col-sm-12">
+                  <br>
+                  <br>
                     <form class="forms-sample" method="post" action="/panel/server_backup">
                       {% raw xsrf_form_html() %}
                       <input type="hidden" name="id" value="{{ data['server_stats']['server_id']['server_id'] }}">
@@ -54,6 +56,49 @@
                         <label for="server_path">{{ translate('serverBackups', 'maxBackups', data['lang']) }} <small class="text-muted ml-1"> - {{ translate('serverBackups', 'maxBackupsDesc', data['lang']) }}</small> </label>
                         <input type="text" class="form-control" name="max_backups" id="max_backups" value="{{ data['backup_config']['max_backups']  }}" placeholder="{{ translate('serverBackups', 'maxBackups', data['lang']) }}" >
                       </div>
+                      <div class="form-group">
+                        <label for="compress" class="form-check-label ml-4 mb-4"></label>
+                          {% if data['backup_config']['compress'] %}
+                          <input type="checkbox" class="form-check-input" id="compress" name="compress"
+                            checked="" value="True">{{ translate('serverBackups', 'compress', data['lang']) }}
+                          {% else %}
+                          <input type="checkbox" class="form-check-input" id="compress" name="compress"
+                            value="True">{{ translate('serverBackups', 'compress', data['lang']) }}
+                          {% end %}
+                      </div>
+                        <div class="form-group">
+                          <label for="server">{{ translate('serverBackups', 'exclusionsTitle', data['lang']) }} <small> - {{ translate('serverBackups', 'excludedChoose', data['lang']) }}</small></label>
+                          <br>
+                          <button class="btn btn-primary mr-2" id="root_files_button" data-server_path="{{ data['server_stats']['server_id']['path']}}" type="button">{{ translate('serverBackups', 'clickExclude', data['lang']) }}</button>
+                        </div>
+                        <input type="number" class="form-control" name="changed" id="changed" value="0" style="visibility: hidden;"></input>
+                      <div class="modal fade" id="dir_select" tabindex="-1" role="dialog" aria-labelledby="dir_select" aria-hidden="true">
+                        <div class="modal-dialog" role="document">
+                          <div class="modal-content">
+                            <div class="modal-header">
+                              <h5 class="modal-title" id="exampleModalLongTitle">{{ translate('serverBackups', 'excludedChoose', data['lang']) }}</h5>
+                              <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                                <span aria-hidden="true">&times;</span>
+                              </button>
+                            </div>
+                            <div class="modal-body">
+                              <div class="tree-ctx-item" id="main-tree-div" data-path="" style="overflow: scroll; max-height:75%;">
+                                <input type="checkbox" id="main-tree-input" name="root_path" value="" disabled>
+                                  <span id="main-tree" class="files-tree-title tree-caret-down root-dir" data-path="">
+                                    <i class="far fa-folder"></i>
+                                    <i class="far fa-folder-open"></i>
+                                    {{ translate('serverFiles', 'files', data['lang']) }}
+                                    </span>
+                                </input>
+                              </div>
+                            </div>
+                            <div class="modal-footer">
+                              <button type="button" id="modal-cancel" class="btn btn-secondary" data-dismiss="modal">{{ translate('serverBackups', 'cancel', data['lang']) }}</button>
+                              <button type="button" id="modal-okay" data-dismiss="modal" class="btn btn-primary">{{ translate('serverWizard', 'save', data['lang']) }}</button>
+                            </div>
+                          </div>
+                        </div>
+                      </div>
 
                       <button type="submit" class="btn btn-success mr-2">{{ translate('serverBackups', 'save', data['lang']) }}</button>
                       <button type="reset" class="btn btn-light">{{ translate('serverBackups', 'cancel', data['lang']) }}</button>
@@ -61,15 +106,10 @@
                 </div>
 
                 <div class="col-md-6 col-sm-12">
-                    <div class="card">
-                      <div class="card-body">
-                        <h4 class="card-title">{{ translate('serverBackups', 'currentBackups', data['lang']) }}</h4>
-
-                      </div>
-                    </div>
                     <div class="text-center">
 
                       <table class="table table-responsive dataTable" id="backup_table">
+                        <h4 class="card-title">{{ translate('serverBackups', 'currentBackups', data['lang']) }}</h4>
                         <thead>
                         <tr>
                             <th width="10%">{{ translate('serverBackups', 'options', data['lang']) }}</th>
@@ -87,7 +127,7 @@
                                 </a>
                                 <br>
                                 <br>
-                                <button data-file="{{ backup['path'] }}" class="btn btn-danger del_button">
+                                <button data-file="{{ backup['path'] }}" data-backup_path="{{ data['backup_path'] }}" class="btn btn-danger del_button">
                                     <i class="fas fa-trash" aria-hidden="true"></i>
                                     {{ translate('serverBackups', 'delete', data['lang']) }}
                                 </button>
@@ -107,6 +147,20 @@
                     </div>
                 </div>
             </div>
+            <div class="col-md-12 col-sm-12">
+              <br>
+              <br>
+              <div class="card-header header-sm d-flex justify-content-between align-items-center">
+                <h4 class="card-title"><i class="fas fa-server"></i> {{ translate('serverBackups', 'excludedBackups', data['lang']) }}  <small class="text-muted ml-1"></small> </h4>
+              </div>
+              <br>
+              <ul>
+                {% for item in data['exclusions'] %}
+                <li>{{item}}</li>
+                <br>
+                {% end %}
+              </ul>
+            </div>
 
         </div>
       </div>
@@ -116,6 +170,44 @@
 
 
 </div>
+<style>
+  /* Remove default bullets */
+ .tree-view,
+ .tree-nested {
+   list-style-type: none;
+   margin: 0;
+   padding: 0;
+   margin-left: 10px;
+ }
+
+ /* Style the items */
+ .tree-item,
+ .files-tree-title {
+   cursor: pointer;
+   user-select: none; /* Prevent text selection */
+ }
+
+ /* Create the caret/arrow with a unicode, and style it */
+ .tree-caret .fa-folder {
+   display: inline-block;
+ }
+ .tree-caret .fa-folder-open {
+   display: none;
+ }
+
+ /* Rotate the caret/arrow icon when clicked on (using JavaScript) */
+ .tree-caret-down .fa-folder {
+   display: none;
+ }
+ .tree-caret-down .fa-folder-open {
+   display: inline-block;
+ }
+
+ /* Hide the nested list */
+ .tree-nested {
+   display: none;
+ }
+</style>
 <!-- content-wrapper ends -->
 
 {% end %}
@@ -123,7 +215,7 @@
 {% block js %}
 <script>
 
-  const serverId = new URLSearchParams(document.location.search).get('id')
+const server_id = new URLSearchParams(document.location.search).get('id')
 
 
   //used to get cookies from browser - this is part of tornados xsrf protection - it's for extra security
@@ -209,6 +301,7 @@
 
     $( ".del_button" ).click(function() {
       var file_to_del = $(this).data("file");
+      var backup_path = $(this).data('backup_path');
 
       console.log("file to delete is" + file_to_del);
 
@@ -226,8 +319,8 @@
         callback: function (result) {
           console.log(result);
           if (result == true) {
-            var full_path = '{{ data['backup_path'] }}' + '/' + file_to_del;
-            del_backup(full_path, serverId);
+            var full_path = backup_path + '/' + file_to_del;
+            del_backup(full_path, server_id);
           }
         }
       });
@@ -245,13 +338,14 @@
             label: '<i class="fas fa-times"></i> {{ translate("serverBackups", "cancel", data['lang']) }}'
           },
           confirm: {
-            label: '<i class="fas fa-check"></i> {{ translate("serverBackups", "restore", data['lang']) }}'
+            label: '<i class="fas fa-check"></i> {{ translate("serverBackups", "restore", data['lang']) }}',
+          className: 'btn-outline-danger'
           }
         },
         callback: function (result) {
           console.log(result);
           if (result == true) {
-            restore_backup(file_to_restore, serverId);
+            restore_backup(file_to_restore, server_id);
           }
         }
       });
@@ -259,7 +353,141 @@
 
   });
 
+document.getElementById("modal-cancel").addEventListener("click", function(){
+  document.getElementById("root_files_button").classList.remove('clicked');
+  document.getElementById("main-tree-div").innerHTML = '<input type="checkbox" id="main-tree-input" name="root_path" value="" disabled><span id="main-tree" class="files-tree-title tree-caret-down root-dir" data-path=""><i class="far fa-folder"></i><i class="far fa-folder-open"></i>{{ translate("serverFiles", "files", data["lang"]) }}</span></input>'
+})
+
+  document.getElementById("root_files_button").addEventListener("click", function(){
+    if($("#root_files_button").data('server_path') != ""){
+      if(document.getElementById('root_files_button').classList.contains('clicked')){
+        show_file_tree();
+        return;
+      }else{
+        document.getElementById('root_files_button').classList.add('clicked');
+        document.getElementById("changed").value = 1;
+      }
+    path = $("#root_files_button").data('server_path')
+    console.log($("#root_files_button").data('server_path'))
+    var token = getCookie("_xsrf");
+    var dialog = bootbox.dialog({
+      message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
+      closeButton: false
+  });
+
+    $.ajax({
+      type: "POST",
+      headers: {'X-XSRFToken': token},
+      url: '/ajax/backup_select?id='+server_id+'&path='+path,
+    });
+  }else{
+    bootbox.alert("You must input a path before selecting this button");
+  }
+  });
+  if (webSocket) {
+    webSocket.on('send_temp_path', function (data) {
+      setTimeout(function(){
+        var x = document.querySelector('.bootbox');
+        if (x) {
+          x.remove()
+        }
+        var x = document.querySelector('.modal-backdrop');
+        if (x) {
+          x.remove()
+        }
+        document.getElementById('main-tree-input').setAttribute('value', data.path)
+        getTreeView(data.path);
+        show_file_tree();
+
+    }, 5000);
+    });
+  }
+
+  function getTreeView(path) {
+    path = path
+
+    $.ajax({
+      type: "GET",
+      url: '/ajax/get_backup_tree?id='+server_id+'&path='+path,
+      dataType: 'text',
+      success: function(data){
+        console.log("got response:");
+        console.log(data);
+
+        dataArr = data.split('\n');
+        serverDir = dataArr.shift(); // Remove & return first element (server directory)
+        text = dataArr.join('\n');
+
+        try{
+          document.getElementById('main-tree-div').innerHTML += text;
+          document.getElementById('main-tree').parentElement.classList.add("clicked");
+        }catch{
+          document.getElementById('files-tree').innerHTML = text;
+        }
+        
+
+        document.getElementsByClassName('files-tree-title')[0].setAttribute('data-path', serverDir);
+        document.getElementsByClassName('files-tree-title')[0].setAttribute('data-name', 'Files');
+
+      },
+    });
+  }
+  function getToggleMain(event) {
+    path = event.target.parentElement.getAttribute('data-path');
+        document.getElementById("files-tree").classList.toggle("d-block");
+        document.getElementById(path+"span").classList.toggle("tree-caret-down");
+        document.getElementById(path+"span").classList.toggle("tree-caret");
+    }
+  
+
+  function getDirView(event) {
+      path = event.target.parentElement.getAttribute('data-path');
+
+      if (document.getElementById(path).classList.contains('clicked')){
+
+      var toggler = document.getElementById(path+"span");
+
+      if (toggler.classList.contains('files-tree-title')){
+          document.getElementById(path+"ul").classList.toggle("d-block");
+          document.getElementById(path+"span").classList.toggle("tree-caret-down");
+      }
+      return;
+    }else{
+    $.ajax({
+      type: "GET",
+      url: '/ajax/get_backup_dir?id='+server_id+'&path='+path,
+      dataType: 'text',
+      success: function(data){
+        console.log("got response:");
+
+        dataArr = data.split('\n');
+        serverDir = dataArr.shift(); // Remove & return first element (server directory)
+        text = dataArr.join('\n');
+
+        try{
+          document.getElementById(path+"span").classList.add('tree-caret-down');
+          document.getElementById(path).innerHTML += text;
+          document.getElementById(path).classList.add("clicked");
+        }catch{
+          console.log("Bad")
+        }
+
+        var toggler = document.getElementById(path);
+
+          if (toggler.classList.contains('files-tree-title')){
+          document.getElementById(path+"span").addEventListener("click", function caretListener() {
+            document.getElementById(path+"ul").classList.toggle("d-block");
+            document.getElementById(path+"span").classList.toggle("tree-caret-down");
+          });
+        }
+      },
+    });
+  }
+  }
+  function show_file_tree(){
+    $("#dir_select").modal();
+  }
 
 </script>
 
-{% end %}
+{% end %}

app/frontend/templates/panel/server_files.html

@@ -133,7 +133,7 @@
                 </style>
                 <ul class="tree-view">
                   <li>
-                    <div class="tree-ctx-item" data-path="{{ data['server_stats']['server_id']['path'] }}">
+                    <div id="root_dir" class="tree-ctx-item" data-path="{{ data['server_stats']['server_id']['path'] }}">
                       <span id="{{ data['server_stats']['server_id']['path'] }}span" class="files-tree-title tree-caret-down root-dir" data-path="{{ data['server_stats']['server_id']['path'] }}" onclick="getToggleMain(event)">
                         <i class="far fa-folder"></i>
                         <i class="far fa-folder-open"></i>
@@ -650,7 +650,7 @@
   }
   function getTreeView(event) {
 
-    let path = "{{ data['server_stats']['server_id']['path'] }}";
+    const path = $('#root_dir').data('path');;
 
     $.ajax({
       type: "GET",

app/frontend/templates/panel/server_schedule_edit.html

@@ -233,18 +233,18 @@ function startup(){
   try{
     document.getElementById("{{ data['schedule']['interval_type'] }}").setAttribute('selected', true);
   }catch{
-    console.log("no element named {{ data['schedule']['interval_type'] }}")
+    console.log("no element named")
   }
   try{
   document.getElementById("{{ data['schedule']['difficulty'] }}").setAttribute('selected', true);
 }catch{
-  console.log("no element named {{ data['schedule']['difficulty'] }}")
+  console.log("no element named")
 }
 try{
   document.getElementById("{{ data['schedule']['action'] }}").setAttribute('selected', true);
 
 }catch{
-  console.log("no element named {{ data['schedule']['action'] }}")
+  console.log("no element named")
 }
   ifDays();
   yesnoCheck();

app/frontend/templates/server/bedrock_wizard.html

@@ -310,9 +310,6 @@
       message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
       closeButton: false
   });
-  setTimeout(function(){
-    dialog.modal('hide');
-}, 5000);
     $.ajax({
       type: "POST",
       headers: {'X-XSRFToken': token},
@@ -433,9 +430,20 @@ function hide(event) {
     }
     if (webSocket) {
       webSocket.on('send_temp_path', function (data) {
-        document.getElementById('main-tree-input').setAttribute('value', data.path)
-        getTreeView(data.path);
-        show_file_tree();
+        setTimeout(function(){
+          var x = document.querySelector('.bootbox');
+          if (x) {
+            x.remove()
+          }
+          var x = document.querySelector('.modal-backdrop');
+          if (x) {
+            x.remove()
+          }
+          document.getElementById('main-tree-input').setAttribute('value', data.path)
+          getTreeView(data.path);
+          show_file_tree();
+  
+      }, 5000);
       });
     }
 

app/frontend/templates/server/wizard.html

@@ -437,9 +437,6 @@
       message: '<p class="text-center mb-0"><i class="fa fa-spin fa-cog"></i> Please wait while we gather your files...</p>',
       closeButton: false
   });
-  setTimeout(function(){
-    dialog.modal('hide');
-}, 5000);
     $.ajax({
       type: "POST",
       headers: {'X-XSRFToken': token},
@@ -587,9 +584,20 @@ function hide(event) {
     }
     if (webSocket) {
       webSocket.on('send_temp_path', function (data) {
-        document.getElementById('main-tree-input').setAttribute('value', data.path)
-        getTreeView(data.path);
-        show_file_tree();
+        setTimeout(function(){
+          var x = document.querySelector('.bootbox');
+          if (x) {
+            x.remove()
+          }
+          var x = document.querySelector('.modal-backdrop');
+          if (x) {
+            x.remove()
+          }
+          document.getElementById('main-tree-input').setAttribute('value', data.path)
+          getTreeView(data.path);
+          show_file_tree();
+  
+      }, 5000);
       });
     }
 

app/migrations/20220227162446_backup_options.py

@@ -0,0 +1,8 @@
+# Generated by database migrator
+import peewee
+
+def migrate(migrator, db):
+    migrator.rename_column('backups', 'directories', 'excluded_dirs')
+
+def rollback(migrator, db):
+    migrator.rename_column('backups', 'excluded_dirs', 'directories')

app/migrations/20220302_compress_backups.py

@@ -0,0 +1,16 @@
+# Generated by database migrator
+import peewee
+
+def migrate(migrator, database, **kwargs):
+    migrator.add_columns('backups', compress=peewee.BooleanField(default=False))
+    """
+    Write your migrations here.
+    """
+
+
+
+def rollback(migrator, database, **kwargs):
+    migrator.drop_columns('backups', ['compress'])
+    """
+    Write your rollback migrations here.
+    """

app/translations/en_EN.json

@@ -18,7 +18,8 @@
         "eulaMsg": "You must agree to the EULA. A copy of the Mojang EULA is linked under this message.",
         "eulaAgree": "Do you agree?",
         "noJava": "Server {} failed to start with error code: We have detected Java is not installed. Please install java then start the server.",
-        "not-downloaded": "We can't seem to find your executable file. Has it finished downloading? Are the permissions set to executable?"
+        "not-downloaded": "We can't seem to find your executable file. Has it finished downloading? Are the permissions set to executable?",
+        "no-file": "We can't seem to locate the requested file. Double check the path. Does Crafty have proper permissions?"
     },
     "404": {
         "contact": "Contact Crafty Control Support via Discord",
@@ -88,7 +89,7 @@
         "allServers": "All Servers",
         "server": "Server",
         "actions": "Actions",
-        "world": "World",
+        "size": "Server Dir Size",
         "motd": "MOTD",
         "version": "Version",
         "status": "Status",
@@ -213,7 +214,9 @@
         "logout": "Logout",
         "preparingLogs": " Please wait while we prepare your logs... We`ll send a notification when they`re ready. This may take a while for large deployments.",
         "downloadLogs": "Download Support Logs?",
-        "finishedPreparing": "We've finished preparing your support logs. Please click download to download"
+        "finishedPreparing": "We've finished preparing your support logs. Please click download to download",
+        "backupComplete": "Backup completed successfully for server ",
+        "backupStarted": "Backup started for server "
     },
     "serverBackups": {
         "backupNow": "Backup Now!",
@@ -236,7 +239,12 @@
         "options": "Options",
         "restoring": "Restoring Backup. This may take a while. Please be patient.",
         "restore": "Restore",
-        "confirmRestore": "Are you sure you want to restore from this backup. All current server files will changed to backup state and will be unrecoverable."
+        "confirmRestore": "Are you sure you want to restore from this backup. All current server files will changed to backup state and will be unrecoverable.",
+        "excludedBackups": "Excluded Paths: ",
+        "excludedChoose": "Choose the paths you wish to exclude from your backups",
+        "clickExclude": "Click to select Exclusions",
+        "exclusionsTitle": "Backup Exclusions",
+        "compress": "Compress Backup"
     },
     "serverFiles": {
         "noscript": "The file manager does not work without JavaScript",