Set audit logging to logfile instead of DB

app/classes/logging/log_formatter.py

@@ -0,0 +1,54 @@
+import logging
+import logging.config
+import json
+from datetime import datetime
+
+
+class JsonEncoderStrFallback(json.JSONEncoder):
+    def default(self, o):
+        try:
+            return super().default(o)
+        except TypeError as exc:
+            if "not JSON serializable" in str(exc):
+                return str(o)
+            raise
+
+
+class JsonEncoderDatetime(JsonEncoderStrFallback):
+    def default(self, o):
+        if isinstance(o, datetime):
+            return o.strftime("%Y-%m-%dT%H:%M:%S%z")
+        else:
+            return super().default(o)
+
+
+class JsonFormatter(logging.Formatter):
+    def formatTime(self, record, datefmt=None):
+        """
+        Override formatTime to customize the time format.
+        """
+        timestamp = datetime.fromtimestamp(record.created)
+        if datefmt:
+            # Use the specified date format
+            return timestamp.strftime(datefmt)
+        else:
+            # Default date format: YYYY-MM-DD HH:MM:SS,mmm
+            secs = int(record.msecs)
+            return f"{timestamp.strftime('%Y-%m-%d %H:%M:%S')},{secs:03d}"
+
+    def format(self, record):
+        log_data = {
+            "level": record.levelname,
+            "time": self.formatTime(record),
+            "log_msg": record.getMessage(),
+        }
+
+        # Filter out standard log record attributes and include only custom ones
+        custom_attrs = ["user_name", "user_id", "server_id", "source_ip"]
+        extra_attrs = {
+            key: value for key, value in record.__dict__.items() if key in custom_attrs
+        }
+
+        # Merge extra attributes with log data
+        log_data.update(extra_attrs)
+        return json.dumps(log_data)

app/classes/models/management.py

@@ -20,6 +20,7 @@ from app.classes.shared.main_models import DatabaseShortcuts
 from app.classes.shared.websocket_manager import WebSocketManager
 
 logger = logging.getLogger(__name__)
+auth_logger = logging.getLogger("audit_log")
 
 
 # **********************************************************************************
@@ -166,50 +167,26 @@ class HelpersManagement:
                 WebSocketManager().broadcast_user(user, "notification", audit_msg)
             except Exception as e:
                 logger.error(f"Error broadcasting to user {user} - {e}")
-
-        AuditLog.insert(
-            {
-                AuditLog.user_name: user_data["username"],
-                AuditLog.user_id: user_id,
-                AuditLog.server_id: server_id,
-                AuditLog.log_msg: audit_msg,
-                AuditLog.source_ip: source_ip,
-            }
-        ).execute()
-        # deletes records when there's more than 300
-        ordered = AuditLog.select().order_by(+AuditLog.created)
-        for item in ordered:
-            if not self.helper.get_setting("max_audit_entries"):
-                max_entries = 300
-            else:
-                max_entries = self.helper.get_setting("max_audit_entries")
-            if AuditLog.select().count() > max_entries:
-                AuditLog.delete().where(AuditLog.audit_id == item.audit_id).execute()
-            else:
-                return
+        auth_logger.info(
+            str(log_msg),
+            extra={
+                "user_name": user_data["username"],
+                "user_id": user_id,
+                "server_id": server_id,
+                "source_ip": source_ip,
+            },
+        )
 
     def add_to_audit_log_raw(self, user_name, user_id, server_id, log_msg, source_ip):
-        AuditLog.insert(
-            {
-                AuditLog.user_name: user_name,
-                AuditLog.user_id: user_id,
-                AuditLog.server_id: server_id,
-                AuditLog.log_msg: log_msg,
-                AuditLog.source_ip: source_ip,
-            }
-        ).execute()
-        # deletes records when there's more than 300
-        ordered = AuditLog.select().order_by(+AuditLog.created)
-        for item in ordered:
-            # configurable through app/config/config.json
-            if not self.helper.get_setting("max_audit_entries"):
-                max_entries = 300
-            else:
-                max_entries = self.helper.get_setting("max_audit_entries")
-            if AuditLog.select().count() > max_entries:
-                AuditLog.delete().where(AuditLog.audit_id == item.audit_id).execute()
-            else:
-                return
+        auth_logger.info(
+            str(log_msg),
+            extra={
+                "user_name": user_name,
+                "user_id": user_id,
+                "server_id": server_id,
+                "source_ip": source_ip,
+            },
+        )
 
     @staticmethod
     def create_crafty_row():

app/config/logging.json

@@ -14,6 +14,9 @@
     "auth": {
       "format": "%(asctime)s - [AUTH] - %(levelname)s - %(message)s"
     },
+    "audit": {
+      "()": "app.classes.logging.log_formatter.JsonFormatter"
+    },
     "cmd_queue": {
       "format": "%(asctime)s - [CMD_QUEUE] - %(levelname)s - %(message)s"
     }
@@ -70,6 +73,14 @@
       "maxBytes": 10485760,
       "backupCount": 20,
       "encoding": "utf8"
+    },
+    "audit_log_handler": {
+      "class": "logging.handlers.RotatingFileHandler",
+      "formatter": "audit",
+      "filename": "logs/audit.log",
+      "maxBytes": 10485760,
+      "backupCount": 20,
+      "encoding": "utf8"
     }
   },
   "loggers": {
@@ -108,6 +119,12 @@
         "cmd_queue_file_handler"
       ],
       "propagate": false
+    },
+    "audit_log": {
+      "level": "INFO",
+      "handlers": [
+        "audit_log_handler"
+      ]
     }
   }
 }

main.py

@@ -17,6 +17,7 @@ from app.classes.models.users import HelperUsers
 from app.classes.models.management import HelpersManagement
 from app.classes.shared.import_helper import ImportHelpers
 from app.classes.shared.websocket_manager import WebSocketManager
+from app.classes.logging.log_formatter import JsonFormatter
 
 console = Console()
 helper = Helpers()
@@ -284,6 +285,11 @@ def setup_logging(debug=True):
 
             logging.config.dictConfig(logging_config)
 
+            # Apply JSON formatting to the "audit" handler
+            for handler in logging.getLogger().handlers:
+                if handler.name == "audit_log_handler":
+                    handler.setFormatter(JsonFormatter())
+
     else:
         logging.basicConfig(level=logging.DEBUG)
         logging.warning(f"Unable to read logging config from {logging_config_file}")