Merge branch 'refactor/config-json' into 'dev'

Managed config.json refactor

See merge request crafty-controller/crafty-4!485

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### New features
 - Add option to run command before backup. ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/536))
 - Make Config.json editable from panel. ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/532))
+- Managed config.json refector (See MR for details). ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/485))
 ### Bug fixes
 - Fix local java server imports. ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/529))
 - Fix Schedule Restore | Add Backup Config Preservation. ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/533))

app/classes/controllers/management_controller.py

@@ -46,6 +46,14 @@ class ManagementController:
     def get_crafty_api_key():
         return HelpersManagement.get_secret_api_key()
 
+    @staticmethod
+    def set_cookie_secret(key):
+        HelpersManagement.set_cookie_secret(key)
+
+    @staticmethod
+    def add_crafty_row():
+        HelpersManagement.create_crafty_row()
+
     # **********************************************************************************
     #                                   Commands Methods
     # **********************************************************************************

app/classes/models/management.py

@@ -43,6 +43,7 @@ class AuditLog(BaseModel):
 # **********************************************************************************
 class CraftySettings(BaseModel):
     secret_api_key = CharField(default="")
+    cookie_secret = CharField(default="")
     login_photo = CharField(default="login_1.jpg")
     login_opacity = IntegerField(default=100)
 
@@ -204,9 +205,22 @@ class HelpersManagement:
             else:
                 return
 
+    @staticmethod
+    def create_crafty_row():
+        CraftySettings.insert(
+            {
+                CraftySettings.secret_api_key: "",
+                CraftySettings.cookie_secret: "",
+                CraftySettings.login_photo: "login_1.jpg",
+                CraftySettings.login_opacity: 100,
+            }
+        ).execute()
+
     @staticmethod
     def set_secret_api_key(key):
-        CraftySettings.insert(secret_api_key=key).execute()
+        CraftySettings.update({CraftySettings.secret_api_key: key}).where(
+            CraftySettings.id == 1
+        ).execute()
 
     @staticmethod
     def get_secret_api_key():
@@ -215,6 +229,19 @@ class HelpersManagement:
         )
         return settings[0].secret_api_key
 
+    @staticmethod
+    def get_cookie_secret():
+        settings = CraftySettings.select(CraftySettings.cookie_secret).where(
+            CraftySettings.id == 1
+        )
+        return settings[0].cookie_secret
+
+    @staticmethod
+    def set_cookie_secret(key):
+        CraftySettings.update({CraftySettings.cookie_secret: key}).where(
+            CraftySettings.id == 1
+        ).execute()
+
     # **********************************************************************************
     #                                  Config Methods
     # **********************************************************************************

app/classes/shared/helpers.py

@@ -377,6 +377,64 @@ class Helpers:
 
         return default_return
 
+    def set_settings(self, data):
+        try:
+            with open(self.settings_file, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=4)
+
+        except Exception as e:
+            logger.critical(
+                f"Config File Error: Unable to read {self.settings_file} due to {e}"
+            )
+            Console.critical(
+                f"Config File Error: Unable to read {self.settings_file} due to {e}"
+            )
+            return False
+
+        return True
+
+    @staticmethod
+    def get_master_config():
+        # Make changes for users' local config.json files here. As of 4.0.20
+        # Config.json was removed from the repo to make it easier for users
+        # To make non-breaking changes to the file.
+        return {
+            "http_port": 8000,
+            "https_port": 8443,
+            "language": "en_EN",
+            "cookie_expire": 30,
+            "show_errors": True,
+            "history_max_age": 7,
+            "stats_update_frequency": 30,
+            "delete_default_json": False,
+            "show_contribute_link": True,
+            "virtual_terminal_lines": 70,
+            "max_log_lines": 700,
+            "max_audit_entries": 300,
+            "disabled_language_files": [],
+            "stream_size_GB": 1,
+            "keywords": ["help", "chunk"],
+            "allow_nsfw_profile_pictures": False,
+            "enable_user_self_delete": False,
+            "reset_secrets_on_next_boot": False,
+        }
+
+    def get_all_settings(self):
+        try:
+            with open(self.settings_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+        except Exception as e:
+            data = {}
+            logger.critical(
+                f"Config File Error: Unable to read {self.settings_file} due to {e}"
+            )
+            Console.critical(
+                f"Config File Error: Unable to read {self.settings_file} due to {e}"
+            )
+
+        return data
+
     @staticmethod
     def is_subdir(server_path, root_dir):
         server_path = os.path.realpath(server_path)

app/classes/shared/main_controller.py

@@ -244,6 +244,41 @@ class Controller:
                 exec_user["user_id"], "support_status_update", results
             )
 
+    def get_config_diff(self):
+        master_config = Helpers.get_master_config()
+        try:
+            user_config = self.helper.get_all_settings()
+        except:
+            # Call helper to set updated config.
+            Console.warning("No Config found. Setting Default Config.json")
+            user_config = master_config
+            keys = list(user_config.keys())
+            keys.sort()
+            sorted_data = {i: user_config[i] for i in keys}
+            self.helper.set_settings(user_config)
+            return
+        items_to_del = []
+
+        # Iterate through user's config.json and check for
+        # Keys/values that need to be removed
+        for key in user_config:
+            if key not in master_config.keys():
+                items_to_del.append(key)
+
+        # Remove key/values from user config that were staged
+        for item in items_to_del[:]:
+            del user_config[item]
+
+        # Add new keys to user config.
+        for key, value in master_config.items():
+            if key not in user_config.keys():
+                user_config[key] = value
+        # Call helper to set updated config.
+        keys = list(user_config.keys())
+        keys.sort()
+        sorted_data = {i: user_config[i] for i in keys}
+        self.helper.set_settings(sorted_data)
+
     def send_log_status(self):
         try:
             return self.log_stats

app/classes/shared/main_models.py

@@ -8,9 +8,10 @@ logger = logging.getLogger(__name__)
 
 
 class DatabaseBuilder:
-    def __init__(self, database, helper, users_helper):
+    def __init__(self, database, helper, users_helper, management_helper):
         self.database = database
         self.helper = helper
+        self.management_helper = management_helper
         self.users_helper = users_helper
 
     def default_settings(self):
@@ -29,6 +30,8 @@ class DatabaseBuilder:
             manager=None,
         )
 
+        self.management_helper.create_crafty_row()
+
     def is_fresh_install(self):
         try:
             num_user = self.users_helper.get_user_total()

app/classes/web/panel_handler.py

@@ -1774,8 +1774,11 @@ class PanelHandler(BaseHandler):
                                 data[key] = False
                             else:
                                 data[key] = arg_data
+                keys = list(data.keys())
+                keys.sort()
+                sorted_data = {i: data[i] for i in keys}
                 with open(self.helper.settings_file, "w", encoding="utf-8") as f:
-                    json.dump(data, f, indent=4)
+                    json.dump(sorted_data, f, indent=4)
             except Exception as e:
                 logger.critical(
                     "Config File Error: Unable to read "

app/classes/web/tornado_handler.py

@@ -11,6 +11,7 @@ import tornado.escape
 import tornado.locale
 import tornado.httpserver
 
+from app.classes.models.management import HelpersManagement
 from app.classes.shared.console import Console
 from app.classes.shared.helpers import Helpers
 from app.classes.shared.main_controller import Controller
@@ -110,10 +111,13 @@ class Webserver:
         https_port = self.helper.get_setting("https_port")
 
         debug_errors = self.helper.get_setting("show_errors")
-        cookie_secret = self.helper.get_setting("cookie_secret")
+        try:
-
+            cookie_secret = HelpersManagement.get_cookie_secret()
-        if cookie_secret is False:
+        except:
+            cookie_secret = False
+        if cookie_secret is False or cookie_secret == "":
             cookie_secret = self.helper.random_string_generator(32)
+            HelpersManagement.set_cookie_secret(cookie_secret)
 
         if not http_port:
             http_port = 8000

app/config/config.json

@@ -1,27 +0,0 @@
-{
-    "http_port": 8000,
-    "https_port": 8443,
-    "language": "en_EN",
-    "cookie_expire": 30,
-    "cookie_secret": "random",
-    "apikey_secret": "random",
-    "show_errors": true,
-    "history_max_age": 7,
-    "stats_update_frequency": 30,
-    "delete_default_json": false,
-    "show_contribute_link": true,
-    "virtual_terminal_lines": 70,
-    "max_log_lines": 700,
-    "max_audit_entries": 300,
-    "disabled_language_files": [
-        "lol_EN.json",
-        ""
-    ],
-    "stream_size_GB": 1,
-    "keywords": [
-        "help",
-        "chunk"
-    ],
-    "allow_nsfw_profile_pictures": false,
-    "enable_user_self_delete": false
-}

app/frontend/templates/panel/config_json.html

@@ -48,7 +48,11 @@
             {% raw xsrf_form_html() %}
 
             {% for item in data['config-json'].items() %}
+            {% if item[0] == "reset_secrets_on_next_boot" %}
+            <div class="form-group" style="background: rgba(243, 21, 6, 0.3); outline: 1px solid red; padding: 10px;">
+            {% else %}
             <div class="form-group">
+            {% end %}
               <label class="form" for="{{item[0]}}">{{item[0]}}
                 <small class="text-muted ml-1">
                 </small> </label><br />
@@ -81,21 +85,19 @@
               {% elif isinstance(item[1], list) %}
               <textarea value="{{','.join(item[1])}}" type="text" name="{{item[0]}}" class="form-control list">{{','.join(item[1])}}</textarea>
               {% elif isinstance(item[1], bool) %}
-              {% if item[1] == True %}
               <div style="margin-left: 30px;">
+              {% if item[1] == True %}
                 <input type="radio" class="form-check-input" name="{{item[0]}}" id="True" value="True" checked>
                  <label for="True">True</label><br>
                 <input type="radio" class="form-check-input" name="{{item[0]}}" id="False" value="False">
                  <label for="False">False</label>
-              </div>
               {% else %}
-              <div style="margin-left: 30px;">
                 <input type="radio" class="form-check-input" name="{{item[0]}}" id="True" value="True">
                  <label for="True">True</label><br>
                 <input type="radio" class="form-check-input" name="{{item[0]}}" id="False" value="False" checked>
                  <label for="False">False</label>
-              </div>
               {% end %}
+              </div>
               {% elif isinstance(item[1], int) %}
               <input type="number" class="form-control" name="{{item[0]}}" id="{{item[0]}}" value="{{ item[1]  }}" step="1" min="0" required>
               {% else %}

app/migrations/20230129_secrets_shh.py

@@ -0,0 +1,16 @@
+# Generated by database migrator
+import peewee
+
+
+def migrate(migrator, database, **kwargs):
+    migrator.add_columns("crafty_settings", cookie_secret=peewee.CharField(default=""))
+    """
+    Write your migrations here.
+    """
+
+
+def rollback(migrator, database, **kwargs):
+    migrator.drop_columns("crafty_settings", ["cookie_secret"])
+    """
+    Write your rollback migrations here.
+    """

docker_launcher.sh

@@ -18,16 +18,6 @@ if [ ! "$(ls -A --ignore=.gitkeep ./app/config)" ]; then
 else
     # Keep version file up to date with image
     cp -f ./app/config_original/version.json ./app/config/version.json
-
-    # Compare if user's config is different from image, and show differences.
-    echo "\033[36mWrapper | \033[35m🏗️  Checking for config.json changes..."
-    cp -f ./app/config_original/config.json ./app/config/config_image_template
-    if [ "$(diff -q ./app/config/config.json ./app/config/config_image_template)" ]; then
-        echo "\033[36mWrapper | \033[33m👷 We've found differences in your local config, please review!,"
-        echo "\033[36m        | \033[33m   (This could be an outdated config.json)"
-    else
-        echo "\033[36mWrapper | \033[32m✅ Config good! Proceeding..."
-    fi
 fi
 
 

main.py

@@ -14,6 +14,7 @@ from app.classes.shared.import3 import Import3
 from app.classes.shared.console import Console
 from app.classes.shared.helpers import Helpers
 from app.classes.models.users import HelperUsers
+from app.classes.models.management import HelpersManagement
 from app.classes.shared.import_helper import ImportHelpers
 
 console = Console()
@@ -53,6 +54,9 @@ def do_intro():
     """
 
     Console.magenta(intro)
+    if not helper.check_file_exists(helper.settings_file):
+        Console.debug("No settings file detected. Creating one.")
+        helper.set_settings(Helpers.get_master_config())
 
 
 def setup_logging(debug=True):
@@ -121,7 +125,8 @@ if __name__ == "__main__":
 
     # do our installer stuff
     user_helper = HelperUsers(database, helper)
-    installer = DatabaseBuilder(database, helper, user_helper)
+    management_helper = HelpersManagement(database, helper)
+    installer = DatabaseBuilder(database, helper, user_helper, management_helper)
     FRESH_INSTALL = installer.is_fresh_install()
 
     if FRESH_INSTALL:
@@ -135,10 +140,22 @@ if __name__ == "__main__":
         installer.default_settings()
     else:
         Console.debug("Existing install detected")
+    Console.info("Checking for reset secret flag")
+    if helper.get_setting("reset_secrets_on_next_boot"):
+        Console.info("Found Reset")
+        management_helper.set_secret_api_key(str(helper.random_string_generator(64)))
+        management_helper.set_cookie_secret(str(helper.random_string_generator(32)))
+        helper.set_setting("reset_secrets_on_next_boot", False)
+    else:
+        Console.info("No flag found. Secrets are staying")
     file_helper = FileHelpers(helper)
     import_helper = ImportHelpers(helper, file_helper)
     # now the tables are created, we can load the tasks_manager and server controller
     controller = Controller(database, helper, file_helper, import_helper)
+    Console.info("Checking for remote changes to config.json")
+    controller.get_config_diff()
+    Console.info("Remote change complete.")
+
     import3 = Import3(helper, controller)
     tasks_manager = TasksManager(helper, controller)
     tasks_manager.start_webserver()