Merge branch 'bugfix/world-size-calc' into 'dev'

Optimize world size calculation

See merge request crafty-controller/crafty-4!550

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Fix backups failing by correctly using tz objects ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/556))
 - Bump Cryptography/pyOpenSSL for CVE-2023-23931 ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/554))
 - Fix debug logging to only display with the -v (verbose) flag ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/560))
+- Optimize world size calculation ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/550))
 ### Tweaks
 - Cleanup authentication helpers ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/545))
 - Optimize file upload progress WS ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/546))

app/classes/minecraft/stats.py

@@ -210,7 +210,7 @@ class Stats:
         return disk_data
 
     @staticmethod
-    def get_world_size(server_path):
+    def get_server_dir_size(server_path):
         total_size = 0
 
         total_size = Helpers.get_dir_size(server_path)

app/classes/shared/helpers.py

@@ -63,6 +63,7 @@ class Helpers:
         self.servers_dir = os.path.join(self.root_dir, "servers")
         self.backup_path = os.path.join(self.root_dir, "backups")
         self.migration_dir = os.path.join(self.root_dir, "app", "migrations")
+        self.dir_migration = False
 
         self.session_file = os.path.join(self.root_dir, "app", "config", "session.lock")
         self.settings_file = os.path.join(self.root_dir, "app", "config", "config.json")
@@ -422,7 +423,7 @@ class Helpers:
             "cookie_expire": 30,
             "show_errors": True,
             "history_max_age": 7,
-            "stats_update_frequency": 30,
+            "stats_update_frequency_seconds": 30,
             "delete_default_json": False,
             "show_contribute_link": True,
             "virtual_terminal_lines": 70,
@@ -435,6 +436,7 @@ class Helpers:
             "enable_user_self_delete": False,
             "reset_secrets_on_next_boot": False,
             "monitored_mounts": Helpers.get_all_mounts(),
+            "dir_size_poll_freq_minutes": 5,
         }
 
     def get_all_settings(self):

app/classes/shared/main_controller.py

@@ -1010,6 +1010,7 @@ class Controller:
         HelpersManagement.set_master_server_dir(server_dir)
 
     def update_master_server_dir(self, server_dir, user_id):
+        self.helper.dir_migration = True
         move_thread = threading.Thread(
             name="dir_move",
             target=self.t_update_master_server_dir,
@@ -1021,15 +1022,15 @@ class Controller:
         )
         move_thread.start()
 
-    def t_update_master_server_dir(self, server_dir, user_id):
+    def t_update_master_server_dir(self, new_server_path, user_id):
-        server_dir = self.helper.wtol_path(server_dir)
+        new_server_path = self.helper.wtol_path(new_server_path)
         self.helper.websocket_helper.broadcast_page(
             "/panel/panel_config", "move_status", "Checking dir"
         )
         current_master = self.helper.wtol_path(
             HelpersManagement.get_master_server_dir()
         )
-        if current_master == server_dir:
+        if current_master == new_server_path:
             logger.info(
                 "Admin tried to change server dir to current server dir. Canceling..."
             )
@@ -1039,7 +1040,7 @@ class Controller:
                 "done",
             )
             return
-        if self.helper.is_subdir(server_dir, current_master):
+        if self.helper.is_subdir(new_server_path, current_master):
             logger.info(
                 "Admin tried to change server dir to be inside a sub directory of the"
                 " current server dir. This will result in a copy loop."
@@ -1054,7 +1055,7 @@ class Controller:
         self.helper.websocket_helper.broadcast_page(
             "/panel/panel_config", "move_status", "Checking permissions"
         )
-        if not self.helper.ensure_dir_exists(os.path.join(server_dir, "servers")):
+        if not self.helper.ensure_dir_exists(os.path.join(new_server_path, "servers")):
             self.helper.websocket_helper.broadcast_user(
                 user_id,
                 "send_start_error",
@@ -1066,15 +1067,15 @@ class Controller:
             )
             return
         # set the cached serve dir
-        self.helper.servers_dir = server_dir
+        self.helper.servers_dir = new_server_path
         # set DB server dir
-        HelpersManagement.set_master_server_dir(server_dir)
+        HelpersManagement.set_master_server_dir(new_server_path)
         servers = self.servers.get_all_defined_servers()
         # move the servers
         for server in servers:
             server_path = server.get("path")
             new_server_path = os.path.join(
-                server_dir, "servers", server.get("server_uuid")
+                new_server_path, "servers", server.get("server_uuid")
             )
             if os.path.isdir(server_path):
                 self.helper.websocket_helper.broadcast_page(
@@ -1095,17 +1096,17 @@ class Controller:
             # reset executable path
             if current_master in server["executable"]:
                 server_obj.executable = str(server["executable"]).replace(
-                    current_master, server_dir
+                    current_master, new_server_path
                 )
             # reset run command path
             if current_master in server["execution_command"]:
                 server_obj.execution_command = str(server["execution_command"]).replace(
-                    current_master, server_dir
+                    current_master, new_server_path
                 )
             # reset log path
             if current_master in server["log_path"]:
                 server_obj.log_path = str(server["log_path"]).replace(
-                    current_master, server_dir
+                    current_master, new_server_path
                 )
             server_obj.path = new_server_path
             failed = False
@@ -1117,6 +1118,7 @@ class Controller:
             else:
                 self.servers.update_unloaded_server(server_obj)
         self.servers.init_all_servers()
+        self.helper.dir_migration = False
         self.helper.websocket_helper.broadcast_page(
             "/panel/panel_config",
             "move_status",

app/classes/shared/server.py

@@ -140,7 +140,10 @@ class ServerInstance:
             )
             self.tz = ZoneInfo("Europe/London")
         self.server_scheduler = BackgroundScheduler(timezone=str(self.tz))
+        self.dir_scheduler = BackgroundScheduler(timezone=str(self.tz))
         self.server_scheduler.start()
+        self.dir_scheduler.start()
+        self.start_dir_calc_task()
         self.backup_thread = threading.Thread(
             target=self.a_backup_server, daemon=True, name=f"backup_{self.name}"
         )
@@ -305,6 +308,22 @@ class ServerInstance:
         else:
             user_lang = HelperUsers.get_user_lang_by_id(user_id)
 
+        # Checks if user is currently attempting to move global server
+        # dir
+        if self.helper.dir_migration:
+            self.helper.websocket_helper.broadcast_user(
+                user_id,
+                "send_start_error",
+                {
+                    "error": self.helper.translation.translate(
+                        "error",
+                        "migration",
+                        user_lang,
+                    )
+                },
+            )
+            return False
+
         if self.stats_helper.get_import_status() and not forge_install:
             if user_id:
                 self.helper.websocket_helper.broadcast_user(
@@ -1374,6 +1393,20 @@ class ServerInstance:
         for user in server_users:
             self.helper.websocket_helper.broadcast_user(user, "remove_spinner", {})
 
+    def start_dir_calc_task(self):
+        server_dt = HelperServers.get_server_data_by_id(self.server_id)
+        self.server_size = self.stats.get_server_dir_size(server_dt["path"])
+        self.dir_scheduler.add_job(
+            self.calc_dir_size,
+            "interval",
+            minutes=self.helper.get_setting("dir_size_poll_freq_minutes"),
+            id=str(self.server_id) + "_dir_poll",
+        )
+
+    def calc_dir_size(self):
+        server_dt = HelperServers.get_server_data_by_id(self.server_id)
+        self.server_size = self.stats.get_server_dir_size(server_dt["path"])
+
     # **********************************************************************************
     #                               Minecraft Servers Statistics
     # **********************************************************************************
@@ -1471,9 +1504,6 @@ class ServerInstance:
         # get our server object, settings and data dictionaries
         self.reload_server_settings()
 
-        # world data
-        server_path = server["path"]
-
         # process stats
         p_stats = Stats._try_get_process_stats(self.process, self.check_running())
 
@@ -1514,7 +1544,7 @@ class ServerInstance:
                 "mem": p_stats.get("memory_usage", 0),
                 "mem_percent": p_stats.get("mem_percentage", 0),
                 "world_name": server_name,
-                "world_size": Stats.get_world_size(server_path),
+                "world_size": self.server_size,
                 "server_port": server_port,
                 "int_ping_results": int_data,
                 "online": ping_data.get("online", False),
@@ -1532,7 +1562,7 @@ class ServerInstance:
                 "mem": p_stats.get("memory_usage", 0),
                 "mem_percent": p_stats.get("mem_percentage", 0),
                 "world_name": server_name,
-                "world_size": Stats.get_world_size(server_path),
+                "world_size": self.server_size,
                 "server_port": server_port,
                 "int_ping_results": int_data,
                 "online": False,
@@ -1600,7 +1630,6 @@ class ServerInstance:
 
         # world data
         server_name = server_dt["server_name"]
-        server_path = server_dt["path"]
 
         # process stats
         p_stats = Stats._try_get_process_stats(self.process, self.check_running())
@@ -1633,7 +1662,7 @@ class ServerInstance:
                     "mem": p_stats.get("memory_usage", 0),
                     "mem_percent": p_stats.get("mem_percentage", 0),
                     "world_name": server_name,
-                    "world_size": Stats.get_world_size(server_path),
+                    "world_size": self.server_size,
                     "server_port": server_port,
                     "int_ping_results": int_data,
                     "online": ping_data.get("online", False),
@@ -1662,7 +1691,7 @@ class ServerInstance:
                         "mem": p_stats.get("memory_usage", 0),
                         "mem_percent": p_stats.get("mem_percentage", 0),
                         "world_name": server_name,
-                        "world_size": Stats.get_world_size(server_path),
+                        "world_size": self.server_size,
                         "server_port": server_port,
                         "int_ping_results": int_data,
                         "online": ping_data["online"],
@@ -1681,7 +1710,7 @@ class ServerInstance:
                         "mem": p_stats.get("memory_usage", 0),
                         "mem_percent": p_stats.get("mem_percentage", 0),
                         "world_name": server_name,
-                        "world_size": Stats.get_world_size(server_path),
+                        "world_size": self.server_size,
                         "server_port": server_port,
                         "int_ping_results": int_data,
                         "online": False,
@@ -1700,7 +1729,7 @@ class ServerInstance:
                 "mem": p_stats.get("memory_usage", 0),
                 "mem_percent": p_stats.get("mem_percentage", 0),
                 "world_name": server_name,
-                "world_size": Stats.get_world_size(server_path),
+                "world_size": self.server_size,
                 "server_port": server_port,
                 "int_ping_results": int_data,
                 "online": False,

app/classes/shared/tasks.py

@@ -636,7 +636,9 @@ class TasksManager:
             logger.error(f"Task failed with error: {event.exception}")
 
     def start_stats_recording(self):
-        stats_update_frequency = self.helper.get_setting("stats_update_frequency")
+        stats_update_frequency = self.helper.get_setting(
+            "stats_update_frequency_seconds"
+        )
         logger.info(
             f"Stats collection frequency set to {stats_update_frequency} seconds"
         )

app/classes/web/ajax_handler.py

@@ -576,6 +576,8 @@ class AjaxHandler(BaseHandler):
             return
 
         elif page == "update_server_dir":
+            if self.helper.dir_migration:
+                return
             for server in self.controller.servers.get_all_servers_stats():
                 if server["stats"]["running"]:
                     self.helper.websocket_helper.broadcast_user(

app/translations/en_EN.json

@@ -185,7 +185,8 @@
         "start-error": "Server {} failed to start with error code: {}",
         "terribleFailure": "What a Terrible Failure!",
         "superError": "You must be a super user to complete this action.",
-        "fileError": "File type must be an image."
+        "fileError": "File type must be an image.",
+        "migration": "Crafty's main server storage is being mirgated to a new location. All server starts have been suspended during this time. Please wait while we finish this migration"
     },
     "footer": {
         "allRightsReserved": "All rights reserved",