Merge branch 'dev' into tweak/reaction-schedules

CHANGELOG.md

@@ -3,9 +3,16 @@
 ### New features
 TBD
 ### Bug fixes
-TBD
+- Fix bug where trying to reconfigure unloaded server would stack ([Commit](https://gitlab.com/crafty-controller/crafty-4/-/commit/1b2fef06fb3b02b76c9506caf7e07e932df95fab) | [Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/460))
+- Fix traceback error when a user click the roles config tab while already on the roles config page; **this is for new role creation only** ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/452))
+- Fix logic issue when removing items from backup exclusions ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/453))
+- Cleanup various JS errors ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/455))
+- Temp fix for `&` issue in pathing and minecraft colour codes ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/457))
+- Cache Gravatar pfp's as to not query every page load ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/459))
+- Fix crash on client list changing while sending websockets ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/461))
 ### Tweaks
-TBD
+- Add button to scroll to bottom of vterm ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/454))
+- Persist schedules and execution commands across backup restores ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/458))
 ### Lang
 TBD
 <br><br>

app/classes/controllers/users_controller.py

@@ -147,14 +147,24 @@ class UsersController:
         return HelperServers.get_total_owned_servers(exec_user_id)
 
     def update_user(self, user_id: str, user_data=None, user_crafty_data=None):
+        # check if user crafty perms were updated
         if user_crafty_data is None:
             user_crafty_data = {}
+        # check if general user data was updated
         if user_data is None:
             user_data = {}
+        # get current user data
         base_data = HelperUsers.get_user(user_id)
         up_data = {}
+        # check if we updated user email. If so we update gravatar
+        if user_data["email"]:
+            pfp = self.helper.get_gravatar_image(user_data["email"])
+            up_data["pfp"] = pfp
+        # create sets to store role data
         added_roles = set()
         removed_roles = set()
+
+        # search for changes in user data
         for key in user_data:
             if key == "user_id":
                 continue
@@ -174,8 +184,10 @@ class UsersController:
                 up_data["hints"] = user_data["hints"]
             elif base_data[key] != user_data[key]:
                 up_data[key] = user_data[key]
+        # change last update for user
         up_data["last_update"] = self.helper.get_time_as_string()
         logger.debug(f"user: {user_data} +role:{added_roles} -role:{removed_roles}")
+
         for role in added_roles:
             HelperUsers.get_or_create(user_id=user_id, role_id=role)
         permissions_mask = user_crafty_data.get("permissions_mask", "000")

app/classes/models/users.py

@@ -42,6 +42,7 @@ class Users(BaseModel):
     preparing = BooleanField(default=False)
     hints = BooleanField(default=True)
     manager = IntegerField(default=None, null=True)
+    pfp = CharField(default="/static/assets/images/faces-clipart/pic-3.png")
 
     class Meta:
         table_name = "users"
@@ -220,6 +221,7 @@ class HelperUsers:
                 Users.password: pw_enc,
                 Users.email: email,
                 Users.enabled: enabled,
+                Users.pfp: self.helper.get_gravatar_image(email),
                 Users.superuser: superuser,
                 Users.created: Helpers.get_time_as_string(),
                 Users.manager: manager,

app/classes/shared/file_helpers.py

@@ -226,18 +226,24 @@ class FileHelpers:
                 comment, "utf-8"
             )  # comments over 65535 bytes will be truncated
             for root, dirs, files in os.walk(path_to_zip, topdown=True):
-                for l_dir in dirs:
+                for l_dir in dirs[:]:
+                    # make all paths in exclusions a unix style slash
+                    # to match directories.
                     if str(os.path.join(root, l_dir)).replace("\\", "/") in ex_replace:
                         dirs.remove(l_dir)
                 ziproot = path_to_zip
+                # iterate through list of files
                 for file in files:
+                    # check if file/dir is in exclusions list.
+                    # Only proceed if not exluded.
                     if (
                         str(os.path.join(root, file)).replace("\\", "/")
                         not in ex_replace
                         and file != "crafty.sqlite"
                     ):
                         try:
-                            logger.info(f"backing up: {os.path.join(root, file)}")
+                            logger.debug(f"backing up: {os.path.join(root, file)}")
+                            # add trailing slash to zip root dir if not windows.
                             if os.name == "nt":
                                 zip_file.write(
                                     os.path.join(root, file),
@@ -254,12 +260,20 @@ class FileHelpers:
                                 f"Error backing up: {os.path.join(root, file)}!"
                                 f" - Error was: {e}"
                             )
+                    # debug logging for exlusions list
+                    else:
+                        logger.debug(f"Found {file} in exclusion list. Skipping...")
+
+                    # add current file bytes to total bytes.
                     total_bytes += os.path.getsize(os.path.join(root, file))
+                    # calcualte percentage based off total size and current archive size
                     percent = round((total_bytes / dir_bytes) * 100, 2)
+                    # package results
                     results = {
                         "percent": percent,
                         "total_files": self.helper.human_readable_file_size(dir_bytes),
                     }
+                    # send status results to page.
                     self.helper.websocket_helper.broadcast_page_params(
                         "/panel/server_detail",
                         {"id": str(server_id)},

app/classes/shared/helpers.py

@@ -20,6 +20,7 @@ import itertools
 from datetime import datetime
 from socket import gethostname
 from contextlib import redirect_stderr, suppress
+import libgravatar
 from packaging import version as pkg_version
 
 from app.classes.shared.null_writer import NullWriter
@@ -658,6 +659,33 @@ class Helpers:
             return True
         return False
 
+    def get_gravatar_image(self, email):
+        profile_url = "/static/assets/images/faces-clipart/pic-3.png"
+        # http://en.gravatar.com/site/implement/images/#rating
+        if self.get_setting("allow_nsfw_profile_pictures"):
+            rating = "x"
+        else:
+            rating = "g"
+
+        # Get grvatar hash for profile pictures
+        if not self.check_internet() or email != "default@example.com" or email != "":
+            gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(email))
+            url = gravatar.get_image(
+                size=80,
+                default="404",
+                force_default=False,
+                rating=rating,
+                filetype_extension=False,
+                use_ssl=True,
+            )  # + "?d=404"
+            try:
+                if requests.head(url).status_code != 404:
+                    profile_url = url
+            except Exception as e:
+                logger.debug(f"Could not pull resource from Gravatar with error {e}")
+
+        return profile_url
+
     @staticmethod
     def get_file_contents(path: str, lines=100):
 

app/classes/shared/tasks.py

@@ -666,6 +666,12 @@ class TasksManager:
             logger.info(
                 "No updates found! You are on the most up to date Crafty version."
             )
+        logger.info("Refreshing Gravatar PFPs...")
+        for user in HelperUsers.get_all_users():
+            if user.email:
+                HelperUsers.update_user(
+                    user.id, {"pfp": self.helper.get_gravatar_image(user.email)}
+                )
 
     def log_watcher(self):
         self.controller.servers.check_for_old_logs()

app/classes/web/ajax_handler.py

@@ -383,6 +383,8 @@ class AjaxHandler(BaseHandler):
             zip_name = bleach.clean(self.get_argument("zip_file", None))
             svr_obj = self.controller.servers.get_server_obj(server_id)
             server_data = self.controller.servers.get_server_data_by_id(server_id)
+
+            # import the server again based on zipfile
             if server_data["type"] == "minecraft-java":
                 backup_path = svr_obj.backup_path
                 if Helpers.validate_traversal(backup_path, zip_name):
@@ -401,6 +403,27 @@ class AjaxHandler(BaseHandler):
                     self.controller.rename_backup_dir(
                         server_id, new_server_id, new_server["server_uuid"]
                     )
+                    # preserve current schedules
+                    for schedule in self.controller.management.get_schedules_by_server(
+                        server_id
+                    ):
+                        self.controller.management.create_scheduled_task(
+                            new_server_id,
+                            schedule.action,
+                            schedule.interval,
+                            schedule.interval_type,
+                            schedule.start_time,
+                            schedule.command,
+                            schedule.name,
+                            schedule.enabled,
+                        )
+                    # preserve execution command
+                    new_server_obj = self.controller.servers.get_server_obj(
+                        new_server_id
+                    )
+                    new_server_obj.execution_command = server_data["execution_command"]
+                    self.controller.servers.update_server(new_server_obj)
+                    # remove old server's tasks
                     try:
                         self.tasks_manager.remove_all_server_tasks(server_id)
                     except:
@@ -424,6 +447,26 @@ class AjaxHandler(BaseHandler):
                     self.controller.rename_backup_dir(
                         server_id, new_server_id, new_server["server_uuid"]
                     )
+                    # preserve current schedules
+                    for schedule in self.controller.management.get_schedules_by_server(
+                        server_id
+                    ):
+                        self.controller.management.create_scheduled_task(
+                            new_server_id,
+                            schedule.action,
+                            schedule.interval,
+                            schedule.interval_type,
+                            schedule.start_time,
+                            schedule.command,
+                            schedule.name,
+                            schedule.enabled,
+                        )
+                    # preserve execution command
+                    new_server_obj = self.controller.servers.get_server_obj(
+                        new_server_id
+                    )
+                    new_server_obj.execution_command = server_data["execution_command"]
+                    self.controller.servers.update_server(new_server_obj)
                     try:
                         self.tasks_manager.remove_all_server_tasks(server_id)
                     except:

app/classes/web/base_handler.py

@@ -104,7 +104,10 @@ class BaseHandler(tornado.web.RequestHandler):
         strip: bool = True,
     ) -> t.Optional[str]:
         arg = self._get_argument(name, default, self.request.arguments, strip)
-        return self.autobleach(name, arg)
+        bleached = self.autobleach(name, arg)
+        if "&" in str(bleached):
+            bleached = bleached.replace("&", "&")
+        return bleached
 
     def get_arguments(self, name: str, strip: bool = True) -> t.List[str]:
         if not isinstance(strip, bool):

app/classes/web/panel_handler.py

@@ -8,7 +8,6 @@ import logging
 import threading
 import shlex
 import bleach
-import libgravatar
 import requests
 import tornado.web
 import tornado.escape
@@ -331,37 +330,6 @@ class PanelHandler(BaseHandler):
             "superuser": superuser,
         }
 
-        # http://en.gravatar.com/site/implement/images/#rating
-        if self.helper.get_setting("allow_nsfw_profile_pictures"):
-            rating = "x"
-        else:
-            rating = "g"
-
-        # Get grvatar hash for profile pictures
-        if exec_user["email"] != "default@example.com" or "":
-            gravatar = libgravatar.Gravatar(
-                libgravatar.sanitize_email(exec_user["email"])
-            )
-            url = gravatar.get_image(
-                size=80,
-                default="404",
-                force_default=False,
-                rating=rating,
-                filetype_extension=False,
-                use_ssl=True,
-            )  # + "?d=404"
-            try:
-                if requests.head(url).status_code != 404:
-                    profile_url = url
-                else:
-                    profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-            except:
-                profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-        else:
-            profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-
-        page_data["user_image"] = profile_url
-
         if page == "unauthorized":
             template = "panel/denied.html"
 
@@ -549,7 +517,7 @@ class PanelHandler(BaseHandler):
                         "log_path": server_temp_obj["log_path"],
                         "executable": server_temp_obj["executable"],
                         "execution_command": server_temp_obj["execution_command"],
-                        "shutdown_timeout": server_obj["shutdown_timeout"],
+                        "shutdown_timeout": server_temp_obj["shutdown_timeout"],
                         "stop_command": server_temp_obj["stop_command"],
                         "executable_update_url": server_temp_obj[
                             "executable_update_url"
@@ -1732,7 +1700,7 @@ class PanelHandler(BaseHandler):
                 if interval_type == "days":
                     sch_time = bleach.clean(self.get_argument("time", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":
@@ -1748,7 +1716,7 @@ class PanelHandler(BaseHandler):
                 delay = bleach.clean(self.get_argument("delay", None))
                 parent = bleach.clean(self.get_argument("parent", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":
@@ -1768,7 +1736,7 @@ class PanelHandler(BaseHandler):
                     return
                 action = bleach.clean(self.get_argument("action", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":
@@ -1894,7 +1862,7 @@ class PanelHandler(BaseHandler):
                 if interval_type == "days":
                     sch_time = bleach.clean(self.get_argument("time", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":
@@ -1909,7 +1877,7 @@ class PanelHandler(BaseHandler):
                 delay = bleach.clean(self.get_argument("delay", None))
                 parent = bleach.clean(self.get_argument("parent", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":
@@ -1929,7 +1897,7 @@ class PanelHandler(BaseHandler):
                     return
                 action = bleach.clean(self.get_argument("action", None))
                 if action == "command":
-                    command = bleach.clean(self.get_argument("command", None))
+                    command = self.get_argument("command", None)
                 elif action == "start":
                     command = "start_server"
                 elif action == "stop":

app/classes/web/routes/api/users/user/pfp.py

@@ -1,6 +1,4 @@
 import logging
-import libgravatar
-import requests
 from app.classes.web.base_api_handler import BaseApiHandler
 
 logger = logging.getLogger(__name__)
@@ -21,29 +19,5 @@ class ApiUsersUserPfpHandler(BaseApiHandler):
             f'User {auth_data[4]["user_id"]} is fetching the pfp for user {user_id}'
         )
 
-        # http://en.gravatar.com/site/implement/images/#rating
-        if self.helper.get_setting("allow_nsfw_profile_pictures"):
-            rating = "x"
-        else:
-            rating = "g"
-
-        # Get grvatar hash for profile pictures
-        if user["email"] != "default@example.com" or "":
-            gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(user["email"]))
-            url = gravatar.get_image(
-                size=80,
-                default="404",
-                force_default=False,
-                rating=rating,
-                filetype_extension=False,
-                use_ssl=True,
-            )
-            try:
-                requests.head(url).raise_for_status()
-            except requests.HTTPError as e:
-                logger.debug("Gravatar profile picture not found", exc_info=e)
-            else:
-                self.finish_json(200, {"status": "ok", "data": url})
-                return
-
-        self.finish_json(200, {"status": "ok", "data": None})
+        self.finish_json(200, {"status": "ok", "data": user["pfp"]})
+        return

app/classes/web/server_handler.py

@@ -5,8 +5,6 @@ import time
 import tornado.web
 import tornado.escape
 import bleach
-import libgravatar
-import requests
 
 from app.classes.models.crafty_permissions import EnumPermissionsCrafty
 from app.classes.shared.helpers import Helpers
@@ -133,34 +131,6 @@ class ServerHandler(BaseHandler):
             "superuser": superuser,
         }
 
-        if self.helper.get_setting("allow_nsfw_profile_pictures"):
-            rating = "x"
-        else:
-            rating = "g"
-
-        if exec_user["email"] != "default@example.com" or "":
-            gravatar = libgravatar.Gravatar(
-                libgravatar.sanitize_email(exec_user["email"])
-            )
-            url = gravatar.get_image(
-                size=80,
-                default="404",
-                force_default=False,
-                rating=rating,
-                filetype_extension=False,
-                use_ssl=True,
-            )  # + "?d=404"
-            try:
-                if requests.head(url).status_code != 404:
-                    profile_url = url
-                else:
-                    profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-            except:
-                profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-        else:
-            profile_url = "/static/assets/images/faces-clipart/pic-3.png"
-
-        page_data["user_image"] = profile_url
         if superuser:
             page_data["roles"] = list_roles
 

app/classes/web/websocket_helper.py

@@ -27,7 +27,7 @@ class WebSocketHelper:
             f"Sending to {len(self.clients)} clients: "
             f"{json.dumps({'event': event_type, 'data': data})}"
         )
-        for client in self.clients:
+        for client in self.clients[:]:
             try:
                 self.send_message(client, event_type, data)
             except Exception as e:
@@ -91,7 +91,7 @@ class WebSocketHelper:
             f"clients: {json.dumps({'event': event_type, 'data': data})}"
         )
 
-        for client in clients:
+        for client in clients[:]:
             try:
                 self.send_message(client, event_type, data)
             except Exception as e:

app/frontend/templates/base.html

@@ -525,10 +525,6 @@
 
       });
     });
-
-    $(window).unload(function () {
-      jQuery.get("/public/logout")
-    });
   </script>
 
   {% block js %}

app/frontend/templates/notify.html

@@ -18,10 +18,10 @@
 
     <li class="nav-item dropdown user-dropdown">
       <a class="nav-link dropdown-toggle" id="UserDropdown" href="#" data-toggle="dropdown" aria-expanded="false">
-        <img class="img-xs rounded-circle profile-picture" src="{{ data['user_image'] }}" alt="Profile image"> </a>
+        <img class="img-xs rounded-circle profile-picture" onerror="pfpError(this)" src="{{ data['user_data']['pfp'] }}" alt="Profile image"> </a>
       <div class="dropdown-menu dropdown-menu-right navbar-dropdown" aria-labelledby="UserDropdown">
         <div class="dropdown-header text-center">
-          <img class="img-md rounded-circle profile-picture" src="{{ data['user_image'] }}" alt="Profile image">
+          <img class="img-md rounded-circle profile-picture" onerror="pfpError(this)" src="{{ data['user_data']['pfp'] }}" alt="Profile image">
           <p class="mb-1 mt-3 font-weight-semibold">{{ data['user_data']['username'] }}</p>
           <p class="font-weight-light text-muted mb-0">Roles: </p>
           {% for r in data['user_role'] %}            
@@ -46,4 +46,12 @@
         <a class="dropdown-item" href="/public/logout"><i class="dropdown-item-icon mdi mdi-power text-primary"></i>{{ translate('notify', 'logout', data['lang']) }}</a>
       </div>
     </li>
-  </ul>
+  </ul>
+
+  <script>
+    function pfpError(image) {
+    image.onerror = "";
+    image.src = "/static/assets/images/faces-clipart/pic-3.png";
+    return true;
+  }
+  </script>

app/frontend/templates/panel/panel_edit_role.html

@@ -39,7 +39,7 @@
         <div class="card-body  pt-0">
           <ul class="nav nav-tabs col-md-12 tab-simple-styled " role="tablist">
             <li class="nav-item">
-              <a class="nav-link active" href="/panel/edit_role?id={{ data['role']['role_id'] }}&subpage=config" role="tab" aria-selected="true">
+              <a class="nav-link active" href="" role="tab" aria-selected="true">
                 <i class="fas fa-cogs"></i>{{ translate('rolesConfig', 'config', data['lang']) }}</a>
             </li>
             <!-- <li class="nav-item">

app/frontend/templates/panel/server_term.html

@@ -40,6 +40,9 @@
           </span>
 
           <div class="col-md-12">
+            <button id="to-bottom" style="visibility: hidden; float: right;" class="btn btn-outline-success">{{ translate('serverDetails', 'reset', data['lang']) }}</button>
+            <br />
+            <br />
             <div class="input-group">
               <div id="virt_console" class="" style="width: 100%; font-size: .8em; padding: 5px 10px; border: 1px solid #383e5d; background-color:#2a2c44;height:500px; overflow: scroll;"></div>
             </div>
@@ -193,8 +196,7 @@
   function new_line_handler(data) {
     $('#virt_console').append(data.line)
     const elem = document.getElementById('virt_console');
-    const scrollDiff = (elem.scrollHeight - elem.scrollTop) - elem.clientHeight;
-    if (!$("#stop_scroll").is(':checked') && scrollDiff < 450) {
+    if (!scrolled) {
       scrollConsole()
     }
   }
@@ -293,6 +295,31 @@
       return nextCommand;
     }
   }
+
+  const chkScroll = (e) => {
+    const elem = $(e.currentTarget);
+
+    if (Math.round(elem[0].scrollHeight - elem.scrollTop()) <= elem.outerHeight()) {
+      document.getElementById("to-bottom").style.visibility = "hidden";
+      scrolled = false;
+    }else{
+      document.getElementById("to-bottom").style.visibility = "visible";
+      scrolled = true;
+    }
+  }
+
+  const scrollToBottom = (id) => {
+   const element = $(`#virt_console`);
+   element.animate({
+      scrollTop: element.prop("scrollHeight")
+   }, 500);
+}
+
+  $(document).ready(() => {
+    var scrolled;
+    $('#virt_console').on('scroll', chkScroll);
+    $('#to-bottom').on('click', scrollToBottom)
+  });
 </script>
 
 {% end %}

app/migrations/20220912_user_pfp.py

@@ -0,0 +1,19 @@
+# Generated by database migrator
+import peewee
+
+
+def migrate(migrator, database, **kwargs):
+    migrator.add_columns(
+        "users",
+        pfp=peewee.CharField(default="/static/assets/images/faces-clipart/pic-3.png"),
+    )
+    """
+    Write your migrations here.
+    """
+
+
+def rollback(migrator, database, **kwargs):
+    migrator.drop_columns("users", ["pfp"])
+    """
+    Write your rollback migrations here.
+    """

app/translations/de_DE.json

@@ -457,7 +457,7 @@
         "absoluteZipPath": "Absoluter Pfad zu dem Server",
         "addRole": "Server zu existierender Rolle hinzufügen",
         "autoCreate": "Wenn keine ausgewählt werden, wird Crafty eine erstellen!",
-        "bePatient": "Bitte haben Sie etwas Geduld, da wir ' + (importing ? 'import' :  'download')",
+        "bePatient": "Bitte haben Sie etwas Geduld, da wir ' + (importing ? 'import' :  'download') + '",
         "buildServer": "Server erstellen!",
         "clickRoot": "Hier klicken, um das Stammverzeichnis auszuwählen",
         "close": "Schließen",

app/translations/en_EN.json

@@ -359,7 +359,8 @@
         "schedule": "Schedule",
         "serverDetails": "Server Details",
         "terminal": "Terminal",
-        "metrics": "Metrics"
+        "metrics": "Metrics",
+        "reset": "Reset Scroll"
     },
     "serverFiles": {
         "clickUpload": "Click here to select your files",