Merge branch 'improve-session-lock' into 'dev'

Improve session.lock checking. Add option to disable language files. Add option to compress backups

See merge request crafty-controller/crafty-commander!178

app/classes/controllers/management_controller.py

@@ -104,8 +104,8 @@ class Management_Controller:
         return management_helper.get_backup_config(server_id)
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None):
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False,):
-        return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs)
+        return management_helper.set_backup_config(server_id, backup_path, max_backups, excluded_dirs, compress)
 
     @staticmethod
     def get_excluded_backup_dirs(server_id: int):

app/classes/minecraft/stats.py

@@ -1,4 +1,3 @@
-import os
 import json
 import logging
 import datetime

app/classes/models/management.py

@@ -128,6 +128,7 @@ class Backups(Model):
     excluded_dirs = CharField(null=True)
     max_backups = IntegerField()
     server_id = ForeignKeyField(Servers, backref='backups_server')
+    compress = BooleanField(default=False)
     class Meta:
         table_name = 'backups'
         database = database
@@ -313,19 +314,21 @@ class helpers_management:
                 "backup_path": row.server_id.backup_path,
                 "excluded_dirs": row.excluded_dirs,
                 "max_backups": row.max_backups,
-                "server_id": row.server_id.server_id
+                "server_id": row.server_id.server_id,
+                "compress": row.compress
             }
         except IndexError:
             conf = {
                 "backup_path": None,
                 "excluded_dirs": None,
                 "max_backups": 0,
-                "server_id": server_id
+                "server_id": server_id,
+                "compress": False,
             }
         return conf
 
     @staticmethod
-    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None):
+    def set_backup_config(server_id: int, backup_path: str = None, max_backups: int = None, excluded_dirs: list = None, compress: bool = False):
         logger.debug(f"Updating server {server_id} backup config with {locals()}")
         if Backups.select().where(Backups.server_id == server_id).count() != 0:
             new_row = False
@@ -334,7 +337,8 @@ class helpers_management:
             conf = {
                 "excluded_dirs": None,
                 "max_backups": 0,
-                "server_id":   server_id
+                "server_id":   server_id,
+                "compress": False
             }
             new_row = True
         if max_backups is not None:
@@ -342,6 +346,7 @@ class helpers_management:
         if excluded_dirs is not None:
             dirs_to_exclude = ",".join(excluded_dirs)
             conf['excluded_dirs'] = dirs_to_exclude
+        conf['compress'] = compress
         if not new_row:
             with database.atomic():
                 if backup_path is not None:

app/classes/shared/file_helpers.py

@@ -9,7 +9,7 @@ from app.classes.shared.console import console
 logger = logging.getLogger(__name__)
 
 try:
-    from zipfile import ZipFile
+    from zipfile import ZipFile, ZIP_DEFLATED
 
 except ModuleNotFoundError as err:
     logger.critical(f"Import Error: Unable to load {err.name} module", exc_info=True)
@@ -87,4 +87,25 @@ class FileHelpers:
 
         return True
 
+    @staticmethod
+    def make_compressed_archive(path_to_destination, path_to_zip):
+        # create a ZipFile object
+        path_to_destination += '.zip'
+        with ZipFile(path_to_destination, 'w', ZIP_DEFLATED) as z:
+            for root, _dirs, files in os.walk(path_to_zip, topdown=True):
+                ziproot = path_to_zip
+                for file in files:
+                    try:
+                        logger.info(f"backing up: {os.path.join(root, file)}")
+                        if os.name == "nt":
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, ""), file))
+                        else:
+                            z.write(os.path.join(root, file), os.path.join(root.replace(ziproot, "/"), file))
+
+                    except Exception as e:
+                        logger.warning(f"Error backing up: {os.path.join(root, file)}! - Error was: {e}")
+
+
+        return True
+
 file_helper = FileHelpers()

app/classes/shared/helpers.py

@@ -17,6 +17,7 @@ import ctypes
 from datetime import datetime
 from socket import gethostname
 from contextlib import suppress
+import psutil
 from requests import get
 
 from app.classes.web.websocket_helper import websocket_helper
@@ -481,6 +482,13 @@ class Helpers:
                 pid = data.get('pid')
                 started = data.get('started')
                 console.critical(f"Another Crafty Controller agent seems to be running...\npid: {pid} \nstarted on: {started}")
+                if psutil.pid_exists(pid):
+                    logger.critical("Found running crafty process. Exiting.")
+                    sys.exit(1)
+                else:
+                    logger.info("No process found for pid. Assuming crafty crashed. Deleting stale session.lock")
+                    os.remove(self.session_file)
+
             except Exception as e:
                 logger.error(f"Failed to locate existing session.lock with error: {e} ")
                 console.error(f"Failed to locate existing session.lock with error: {e} ")

app/classes/shared/server.py

@@ -604,6 +604,11 @@ class Server:
                 else:
                     # If not, just remove the file
                     os.remove(excluded_dir)
+            if conf['compress']:
+                logger.debug("Found compress backup to be true. Calling compressed archive")
+                file_helper.make_compressed_archive(helper.get_os_understandable_path(backup_filename), tempDir)
+            else:
+                logger.debug("Found compress backup to be false. Calling NON-compressed archive")
                 file_helper.make_archive(helper.get_os_understandable_path(backup_filename), tempDir)
 
             while len(self.list_backups()) > conf["max_backups"] and conf["max_backups"] > 0:

app/classes/web/panel_handler.py

@@ -606,8 +606,9 @@ class PanelHandler(BaseHandler):
                 page_data['super-disabled'] = ''
             else:
                 page_data['super-disabled'] = 'disabled'
-            for file in os.listdir(os.path.join(helper.root_dir, 'app', 'translations')):
+            for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))):
                 if file.endswith('.json'):
+                    if file not in helper.get_setting('disabled_language_files'):
                         if file != str(page_data['languages'][0] + '.json'):
                             page_data['languages'].append(file.split('.')[0])
 
@@ -738,6 +739,7 @@ class PanelHandler(BaseHandler):
 
             for file in sorted(os.listdir(os.path.join(helper.root_dir, 'app', 'translations'))):
                 if file.endswith('.json'):
+                    if file not in helper.get_setting('disabled_language_files'):
                         if file != str(page_data['languages'][0] + '.json'):
                             page_data['languages'].append(file.split('.')[0])
 
@@ -1059,6 +1061,7 @@ class PanelHandler(BaseHandler):
             logger.debug(self.request.arguments)
             server_id = self.get_argument('id', None)
             server_obj = self.controller.servers.get_server_obj(server_id)
+            compress = self.get_argument('compress', False)
             check_changed = self.get_argument('changed')
             if str(check_changed) == str(1):
                 checked = self.get_body_arguments('root_path')
@@ -1089,7 +1092,7 @@ class PanelHandler(BaseHandler):
             server_obj = self.controller.servers.get_server_obj(server_id)
             server_obj.backup_path = backup_path
             self.controller.servers.update_server(server_obj)
-            self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked)
+            self.controller.management.set_backup_config(server_id, max_backups=max_backups, excluded_dirs=checked, compress=bool(compress))
 
             self.controller.management.add_to_audit_log(exec_user['user_id'],
                                        f"Edited server {server_id}: updated backups",
@@ -1189,7 +1192,9 @@ class PanelHandler(BaseHandler):
                         "start_time": sch_time,
                         "enabled": enabled,
                         "one_time": one_time,
-                        "cron_string": ''
+                        "cron_string": '',
+                        "parent": None,
+                        "delay": 0
                     }
                 elif difficulty == "reaction":
                     job_data = {
@@ -1337,7 +1342,9 @@ class PanelHandler(BaseHandler):
                         "start_time": sch_time,
                         "enabled": enabled,
                         "one_time": one_time,
-                        "cron_string": ''
+                        "cron_string": '',
+                        "parent": None,
+                        "delay": 0
                     }
                 elif difficulty == "advanced":
                     job_data = {

app/config/config.json

@@ -14,6 +14,7 @@
   "virtual_terminal_lines": 70,
   "max_log_lines": 700,
   "max_audit_entries": 300,
+  "disabled_language_files": ["lol_EN.json", ""],
   "keywords": ["help", "chunk"],
   "allow_nsfw_profile_pictures": false
 }

app/frontend/templates/panel/server_backup.html

@@ -55,6 +55,16 @@
                       <div class="form-group">
                         <label for="server_path">{{ translate('serverBackups', 'maxBackups', data['lang']) }} <small class="text-muted ml-1"> - {{ translate('serverBackups', 'maxBackupsDesc', data['lang']) }}</small> </label>
                         <input type="text" class="form-control" name="max_backups" id="max_backups" value="{{ data['backup_config']['max_backups']  }}" placeholder="{{ translate('serverBackups', 'maxBackups', data['lang']) }}" >
+                      </div>
+                      <div class="form-group">
+                        <label for="compress" class="form-check-label ml-4 mb-4"></label>
+                          {% if data['backup_config']['compress'] %}
+                          <input type="checkbox" class="form-check-input" id="compress" name="compress"
+                            checked="" value="True">{{ translate('serverBackups', 'compress', data['lang']) }}
+                          {% else %}
+                          <input type="checkbox" class="form-check-input" id="compress" name="compress"
+                            value="True">{{ translate('serverBackups', 'compress', data['lang']) }}
+                          {% end %}
                       </div>
                         <div class="form-group">
                           <label for="server">{{ translate('serverBackups', 'exclusionsTitle', data['lang']) }} <small> - {{ translate('serverBackups', 'excludedChoose', data['lang']) }}</small></label>

app/migrations/20220302_compress_backups.py

@@ -0,0 +1,16 @@
+# Generated by database migrator
+import peewee
+
+def migrate(migrator, database, **kwargs):
+    migrator.add_columns('backups', compress=peewee.BooleanField(default=False))
+    """
+    Write your migrations here.
+    """
+
+
+
+def rollback(migrator, database, **kwargs):
+    migrator.drop_columns('backups', ['compress'])
+    """
+    Write your rollback migrations here.
+    """

app/translations/en_EN.json

@@ -243,7 +243,8 @@
         "excludedBackups": "Excluded Paths: ",
         "excludedChoose": "Choose the paths you wish to exclude from your backups",
         "clickExclude": "Click to select Exclusions",
-        "exclusionsTitle": "Backup Exclusions"
+        "exclusionsTitle": "Backup Exclusions",
+        "compress": "Compress Backup"
     },
     "serverFiles": {
         "noscript": "The file manager does not work without JavaScript",