Merge branch 'tweak/json-audit-log' into 'dev'

Set audit logging to logfile instead of DB

See merge request crafty-controller/crafty-4!751
This commit is contained in:
Iain Powrie 2024-05-09 20:07:54 +00:00
commit 28ac3d9915
33 changed files with 262 additions and 165 deletions

View File

@ -8,6 +8,7 @@ TBD
- Reset query arguments on login if `?next` is not available ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/750))
### Tweaks
- Add link to go back to dashboard on error page ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/743))
- Set audit logging to logfile instead of DB ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/751))
### Lang
- Changes of phrase in `cs_CS` translation ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/749))
<br><br>

View File

@ -95,9 +95,6 @@ class ManagementController:
# **********************************************************************************
# Audit_Log Methods
# **********************************************************************************
@staticmethod
def get_activity_log():
return HelpersManagement.get_activity_log()
def add_to_audit_log(self, user_id, log_msg, server_id=None, source_ip=None):
return self.management_helper.add_to_audit_log(

View File

@ -0,0 +1,53 @@
import logging
import logging.config
import json
from datetime import datetime
class JsonEncoderStrFallback(json.JSONEncoder):
def default(self, o):
try:
return super().default(o)
except TypeError as exc:
if "not JSON serializable" in str(exc):
return str(o)
raise
class JsonEncoderDatetime(JsonEncoderStrFallback):
def default(self, o):
if isinstance(o, datetime):
return o.strftime("%Y-%m-%dT%H:%M:%S%z")
return super().default(o)
class JsonFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None):
"""
Override formatTime to customize the time format.
"""
timestamp = datetime.fromtimestamp(record.created)
if datefmt:
# Use the specified date format
return timestamp.strftime(datefmt)
# Default date format: YYYY-MM-DD HH:MM:SS,mmm
secs = int(record.msecs)
return f"{timestamp.strftime('%Y-%m-%d %H:%M:%S')},{secs:03d}"
def format(self, record):
log_data = {
"level": record.levelname,
"time": self.formatTime(record),
"log_msg": record.getMessage(),
}
# Filter out standard log record attributes and include only custom ones
custom_attrs = ["user_name", "user_id", "server_id", "source_ip"]
extra_attrs = {
key: value for key, value in record.__dict__.items() if key in custom_attrs
}
# Merge extra attributes with log data
log_data.update(extra_attrs)
return json.dumps(log_data)

View File

@ -16,28 +16,10 @@ from app.classes.models.base_model import BaseModel
from app.classes.models.users import HelperUsers
from app.classes.models.servers import Servers
from app.classes.models.server_permissions import PermissionsServers
from app.classes.shared.main_models import DatabaseShortcuts
from app.classes.shared.websocket_manager import WebSocketManager
logger = logging.getLogger(__name__)
# **********************************************************************************
# Audit_Log Class
# **********************************************************************************
class AuditLog(BaseModel):
audit_id = AutoField()
created = DateTimeField(default=datetime.datetime.now)
user_name = CharField(default="")
user_id = IntegerField(default=0, index=True)
source_ip = CharField(default="127.0.0.1")
server_id = ForeignKeyField(
Servers, backref="audit_server", null=True
) # When auditing global events, use server ID null
log_msg = TextField(default="")
class Meta:
table_name = "audit_log"
auth_logger = logging.getLogger("audit_log")
# **********************************************************************************
@ -149,10 +131,6 @@ class HelpersManagement:
# **********************************************************************************
# Audit_Log Methods
# **********************************************************************************
@staticmethod
def get_activity_log():
query = AuditLog.select()
return DatabaseShortcuts.return_db_rows(query)
def add_to_audit_log(self, user_id, log_msg, server_id=None, source_ip=None):
logger.debug(f"Adding to audit log User:{user_id} - Message: {log_msg} ")
@ -166,50 +144,28 @@ class HelpersManagement:
WebSocketManager().broadcast_user(user, "notification", audit_msg)
except Exception as e:
logger.error(f"Error broadcasting to user {user} - {e}")
AuditLog.insert(
{
AuditLog.user_name: user_data["username"],
AuditLog.user_id: user_id,
AuditLog.server_id: server_id,
AuditLog.log_msg: audit_msg,
AuditLog.source_ip: source_ip,
}
).execute()
# deletes records when there's more than 300
ordered = AuditLog.select().order_by(+AuditLog.created)
for item in ordered:
if not self.helper.get_setting("max_audit_entries"):
max_entries = 300
else:
max_entries = self.helper.get_setting("max_audit_entries")
if AuditLog.select().count() > max_entries:
AuditLog.delete().where(AuditLog.audit_id == item.audit_id).execute()
else:
return
auth_logger.info(
str(log_msg),
extra={
"user_name": user_data["username"],
"user_id": user_id,
"server_id": server_id,
"source_ip": source_ip,
},
)
def add_to_audit_log_raw(self, user_name, user_id, server_id, log_msg, source_ip):
AuditLog.insert(
{
AuditLog.user_name: user_name,
AuditLog.user_id: user_id,
AuditLog.server_id: server_id,
AuditLog.log_msg: log_msg,
AuditLog.source_ip: source_ip,
}
).execute()
# deletes records when there's more than 300
ordered = AuditLog.select().order_by(+AuditLog.created)
for item in ordered:
# configurable through app/config/config.json
if not self.helper.get_setting("max_audit_entries"):
max_entries = 300
else:
max_entries = self.helper.get_setting("max_audit_entries")
if AuditLog.select().count() > max_entries:
AuditLog.delete().where(AuditLog.audit_id == item.audit_id).execute()
else:
return
if isinstance(server_id, Servers) and server_id is not None:
server_id = server_id.server_id
auth_logger.info(
str(log_msg),
extra={
"user_name": user_name,
"user_id": user_id,
"server_id": server_id,
"source_ip": source_ip,
},
)
@staticmethod
def create_crafty_row():

View File

@ -1506,8 +1506,6 @@ class PanelHandler(BaseHandler):
template = "panel/panel_edit_role.html"
elif page == "activity_logs":
page_data["audit_logs"] = self.controller.management.get_activity_log()
template = "panel/activity_logs.html"
elif page == "download_file":

View File

@ -1,3 +1,5 @@
import os
import json
from app.classes.web.base_api_handler import BaseApiHandler
@ -22,9 +24,17 @@ class ApiCraftyLogIndexHandler(BaseApiHandler):
raise NotImplementedError
if log_type == "audit":
with open(
os.path.join(self.controller.project_root, "logs", "audit.log"),
"r",
encoding="utf-8",
) as f:
log_lines = [json.loads(line) for line in f]
rev_log_lines = log_lines[::-1]
return self.finish_json(
200,
{"status": "ok", "data": self.controller.management.get_activity_log()},
{"status": "ok", "data": rev_log_lines},
)
if log_type == "session":

View File

@ -14,6 +14,9 @@
"auth": {
"format": "%(asctime)s - [AUTH] - %(levelname)s - %(message)s"
},
"audit": {
"()": "app.classes.logging.log_formatter.JsonFormatter"
},
"cmd_queue": {
"format": "%(asctime)s - [CMD_QUEUE] - %(levelname)s - %(message)s"
}
@ -70,6 +73,14 @@
"maxBytes": 10485760,
"backupCount": 20,
"encoding": "utf8"
},
"audit_log_handler": {
"class": "logging.handlers.RotatingFileHandler",
"formatter": "audit",
"filename": "logs/audit.log",
"maxBytes": 10485760,
"backupCount": 20,
"encoding": "utf8"
}
},
"loggers": {
@ -108,6 +119,12 @@
"cmd_queue_file_handler"
],
"propagate": false
},
"audit_log": {
"level": "INFO",
"handlers": [
"audit_log_handler"
]
}
}
}

View File

@ -36,25 +36,21 @@
<table class="table table-hover" id="audit_table" style="overflow: scroll;" width="100%">
<thead>
<tr class="rounded">
<td>Username</td>
<td>Time</td>
<td>Action</td>
<td>Server ID</td>
<td>IP</td>
<th>Time</th>
<th>Username</th>
<th>Action</th>
<th>Server ID</th>
<th>IP</th>
</tr>
</thead>
<tbody>
{% for row in data['audit_logs'] %}
<tr>
<td>{{ row['user_name'] }}</td>
<td>
{{ row['created'].strftime('%Y-%m-%d %H:%M:%S') }}
<td colspan="5" id="image-div" class="text-center"> <!-- Center image within table -->
<img class="img-center" id="logo-animate" src="../static/assets/images/crafty-logo-square-1024.png"
alt="Crafty Logo, Crafty is loading" width="20%"><br><br>{{ translate('datatables',
'loadingRecords', data['lang'])}}
</td>
<td>{{ row['log_msg'] }}</td>
<td>{{ row['server_id'] }}</td>
<td>{{ row['source_ip'] }}</td>
</tr>
{% end %}
</tbody>
</table>
@ -79,17 +75,6 @@
{% end %}
{% block js %}
<script>
$(document).ready(function () {
console.log('ready for JS!')
$('#audit_table').DataTable({
'order': [1, 'desc']
}
);
});
</script>
<script>
$(document).ready(function () {
$('[data-toggle="popover"]').popover();
@ -112,6 +97,74 @@
$('.too_small').popover("hide");
} // New width
});
$(document).ready(function () {
console.log('ready for JS!')
// Initialize DataTables
// Load initial data
getActivity();
});
function updateActivity(data) {
let tbody = $('#audit_table tbody');
tbody.empty(); // Clear existing rows
$.each(data, function (index, value) {
let row = $('<tr>');
row.append(`<td>${value.time}</td>`);
if (value.user_name != "system" && value.user_id != "-1") {
row.append(`<td><a href="/panel/edit_user?id=${value.user_id}">${value.user_name}</a></td>`);
} else {
row.append(`<td>${value.user_name}</td>`);
}
row.append(`<td>${value.log_msg}</td>`);
row.append(`<td>${value.server_id}</td>`);
row.append(`<td>${value.source_ip}</td>`);
tbody.append(row);
});
$('#audit_table').DataTable({
'order': [[0, 'desc']], // Sort by the first column in descending order
filter: true,
"searching": true,
})
}
async function getActivity() {
var token = getCookie("_xsrf");
let res = await fetch(`/api/v2/crafty/logs/audit`, {
method: 'GET',
headers: {
'X-XSRFToken': token
},
});
let responseData = await res.json();
console.log(responseData);
if (responseData.status === "ok") {
updateActivity(responseData.data);
console.log("activity update")
} else {
bootbox.alert(responseData.error)
}
}
function rotateImage(degree) {
$('#logo-animate').animate({ transform: degree }, {
step: function (now, fx) {
$(this).css({
'-webkit-transform': 'rotate(' + now + 'deg)',
'-moz-transform': 'rotate(' + now + 'deg)',
'transform': 'rotate(' + now + 'deg)'
});
}
});
setTimeout(function () {
rotateImage(360);
}, 2000);
}
$(document).ready(function () {
setTimeout(function () {
rotateImage(360);
}, 2000);
});
</script>
{% end %}

View File

@ -6,7 +6,6 @@ import logging
from app.classes.shared.console import Console
from app.classes.shared.migration import Migrator, MigrateHistory
from app.classes.models.management import (
AuditLog,
Webhooks,
Schedules,
Backups,
@ -61,17 +60,6 @@ def migrate(migrator: Migrator, database, **kwargs):
peewee.CharField(primary_key=True, default=str(uuid.uuid4())),
)
# Changes on Audit Log Table
migrator.alter_column_type(
AuditLog,
"server_id",
peewee.ForeignKeyField(
Servers,
backref="audit_server",
null=True,
field=peewee.CharField(primary_key=True, default=str(uuid.uuid4())),
),
)
# Changes on Webhook Table
migrator.alter_column_type(
Webhooks,
@ -109,13 +97,6 @@ def rollback(migrator: Migrator, database, **kwargs):
peewee.AutoField(),
)
# Changes on Audit Log Table
migrator.alter_column_type(
AuditLog,
"server_id",
peewee.IntegerField(default=None, index=True),
)
# Changes on Webhook Table
migrator.alter_column_type(
Webhooks,

View File

@ -6,7 +6,6 @@ import logging
from app.classes.shared.console import Console
from app.classes.shared.migration import Migrator, MigrateHistory
from app.classes.models.management import (
AuditLog,
Webhooks,
Schedules,
Backups,
@ -73,20 +72,6 @@ def migrate(migrator: Migrator, database, **kwargs):
try:
logger.info("Migrating Data from Int to UUID (Foreign Keys)")
Console.info("Migrating Data from Int to UUID (Foreign Keys)")
# Changes on Audit Log Table
for audit_log in AuditLog.select():
old_server_id = audit_log.server_id_id
if old_server_id == "0" or old_server_id is None:
server_uuid = None
else:
try:
server = Servers.get_by_id(old_server_id)
server_uuid = server.server_uuid
except:
server_uuid = old_server_id
AuditLog.update(server_id=server_uuid).where(
AuditLog.audit_id == audit_log.audit_id
).execute()
# Changes on Webhooks Log Table
for webhook in Webhooks.select():
@ -247,21 +232,6 @@ def rollback(migrator: Migrator, database, **kwargs):
try:
logger.info("Migrating Data from UUID to Int (Foreign Keys)")
Console.info("Migrating Data from UUID to Int (Foreign Keys)")
# Changes on Audit Log Table
for audit_log in AuditLog.select():
old_server_id = audit_log.server_id_id
if old_server_id is None:
new_server_id = 0
else:
try:
server = Servers.get_or_none(Servers.server_uuid == old_server_id)
new_server_id = server.server_id
except:
new_server_id = old_server_id
AuditLog.update(server_id=new_server_id).where(
AuditLog.audit_id == audit_log.audit_id
).execute()
# Changes on Webhooks Log Table
for webhook in Webhooks.select():
old_server_id = webhook.server_id_id

View File

@ -0,0 +1,34 @@
import peewee
import datetime
from peewee import (
AutoField,
DateTimeField,
CharField,
IntegerField,
ForeignKeyField,
TextField,
)
from app.classes.shared.server import Servers
def migrate(migrator, db):
migrator.drop_table("audit_log")
def rollback(migrator, db):
class AuditLog(peewee.Model):
audit_id = AutoField()
created = DateTimeField(default=datetime.datetime.now)
user_name = CharField(default="")
user_id = IntegerField(default=0, index=True)
source_ip = CharField(default="127.0.0.1")
server_id = ForeignKeyField(
Servers, backref="audit_server", null=True
) # When auditing global events, use server ID null
log_msg = TextField(default="")
class Meta:
table_name = "audit_log"
migrator.create_table(AuditLog)

View File

@ -117,6 +117,7 @@
"welcome": "Vítejte v Crafty Controlleru"
},
"datatables": {
"loadingRecords": "Načítání...",
"i18n": {
"aria": {
"sortAscending": ": aktivace řazení sloupce vzestupně",
@ -693,4 +694,4 @@
"webhook_body": "Webhook Body",
"webhooks": "Webhooky"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Willkommen bei Crafty Controller"
},
"datatables": {
"loadingRecords": "Laden...",
"i18n": {
"aria": {
"sortAscending": ": Aktivieren, um die Spalte aufsteigend zu sortieren",
@ -674,4 +675,4 @@
"webhook_body": "Webhook-Inhalt",
"webhooks": "Webhooks"
}
}
}

View File

@ -116,6 +116,7 @@
"welcome": "Welcome to Crafty Controller"
},
"datatables": {
"loadingRecords": "Loading...",
"i18n": {
"aria": {
"sortAscending": ": activate to sort column ascending",
@ -672,4 +673,4 @@
"webhook_body": "Webhook Body",
"webhooks": "Webhooks"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Bienvenido a Crafty Controller"
},
"datatables": {
"loadingRecords": "Cargando...",
"i18n": {
"aria": {
"sortAscending": ": activar para ordenar las columnas de manera ascendente",
@ -674,4 +675,4 @@
"webhook_body": "Cuerpo del Webhook",
"webhooks": "Webhooks"
}
}
}

View File

@ -100,6 +100,7 @@
"welcome": "Tervetuloa Crafty Controller"
},
"datatables": {
"loadingRecords": "Ladataan...",
"i18n": {
"aria": {
"sortAscending": ": lajittele sarake nousevasti",
@ -560,4 +561,4 @@
"userSettings": "Käyttäjäasetukset",
"uses": "Sallittujen käyttäkertojen määtä (-1 == Ei rajaa)"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Bienvenue sur Crafty Controller"
},
"datatables": {
"loadingRecords": "Chargement ...",
"i18n": {
"aria": {
"sortAscending": ": activer pour trier les colonnes dans l'ordre croissant",
@ -674,4 +675,4 @@
"webhook_body": "Corps du Webhook",
"webhooks": "Webhooks"
}
}
}

View File

@ -99,6 +99,7 @@
"welcome": "Wolkom by Crafty Controller"
},
"datatables": {
"loadingRecords": "Laden...",
"i18n": {
"aria": {
"sortAscending": ": aktivearje om kolom oprinnend te sortearjen",
@ -529,4 +530,4 @@
"userSettings": "Brûkersynstellingen",
"uses": "Oantal gebrûk tastien (-1==Gjin limyt)"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "ברוכים הבאים ל-פאנל קראפטי"
},
"datatables": {
"loadingRecords": "...טוען",
"i18n": {
"aria": {
"sortAscending": ": הפעילו כדי למיין עמודות בסדר עולה",
@ -674,4 +675,4 @@
"webhook_body": "גוף ה-Webhook",
"webhooks": "Webhooks"
}
}
}

View File

@ -99,6 +99,7 @@
"welcome": "Dobrodošli u Crafty Controller"
},
"datatables": {
"loadingRecords": "Učitavanje...",
"i18n": {
"aria": {
"sortAscending": ": aktiviraj za sortiranje stupca uzlazno",
@ -529,4 +530,4 @@
"userSettings": "Korisničke postavke",
"uses": "Broj dopuštenih upotreba (-1==Bez ograničenja)"
}
}
}

View File

@ -100,6 +100,7 @@
"welcome": "Selamat Datang Di Crafty Controller"
},
"datatables": {
"loadingRecords": "Loading...",
"i18n": {
"aria": {
"sortAscending": ": aktifkan untuk mengurutkan kolom menaik",
@ -536,4 +537,4 @@
"userSettings": "Pengaturan Pengguna",
"uses": "Jumlah penggunaan yang diizinkan (-1==No Limit)"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Benvenuto su Crafty Controller"
},
"datatables": {
"loadingRecords": "Carico...",
"i18n": {
"aria": {
"sortAscending": ": attiva per ordinare le colonne in modo ascendente",
@ -674,4 +675,4 @@
"webhook_body": "Corpo del Webhook",
"webhooks": "Webhook"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "WELCOM 2 CWAFTY CONTROLLR"
},
"datatables": {
"loadingRecords": "Loading...",
"i18n": {
"aria": {
"sortAscending": ": activate to sort column ascending",
@ -674,4 +675,4 @@
"webhook_body": "WEBHOOK FISH",
"webhooks": "WEBHOOKZ"
}
}
}

View File

@ -118,6 +118,7 @@
"welcome": "Esiet sveicināts Crafty Controller"
},
"datatables": {
"loadingRecords": "Ielādē...",
"i18n": {
"aria": {
"sortAscending": ": aktivizēt lai kārotu kolonnu augoši",
@ -675,4 +676,4 @@
"webhook_body": "Webhook Saturs",
"webhooks": "Webhooki"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Welkom bij Crafty Controller "
},
"datatables": {
"loadingRecords": "Bezig met laden...",
"i18n": {
"aria": {
"sortAscending": ": activeren om kolom oplopend te sorteren",
@ -674,4 +675,4 @@
"webhook_body": "Webhook-body",
"webhooks": "Webhooks"
}
}
}

View File

@ -99,6 +99,7 @@
"welcome": "Welkom bij Crafty Controller"
},
"datatables": {
"loadingRecords": "Laden...",
"i18n": {
"aria": {
"sortAscending": ": activeer om kolom oplopend te sorteren",
@ -529,4 +530,4 @@
"userSettings": "Gebruikersinstellingen",
"uses": "Aantal keer toegestaan (-1==Geen limiet)"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Witamy w Crafty Controller"
},
"datatables": {
"loadingRecords": "Wczytywanie...",
"i18n": {
"aria": {
"sortAscending": ": aktywuj, aby sortować kolumny w góre",
@ -673,4 +674,4 @@
"webhook_body": "Treść Webhooka",
"webhooks": "Webhooki"
}
}
}

View File

@ -100,6 +100,7 @@
"welcome": "Bem-vindo ao Crafty Controller"
},
"datatables": {
"loadingRecords": "Carregando...",
"i18n": {
"aria": {
"sortAscending": ": ative para ordenar a coluna de forma ascendente",
@ -537,4 +538,4 @@
"userSettings": "Configurações do Usuário",
"uses": "Número de Usos Permitidos (-1==Sem Limite)"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "ยินดีต้อนรับสู่ Crafty Controller"
},
"datatables": {
"loadingRecords": "กำลังโหลด...",
"i18n": {
"aria": {
"sortAscending": ": เปิดใช้งานเพื่อเรียงลำดับคอลัมน์จากน้อยไปมาก",
@ -673,4 +674,4 @@
"webhook_body": "ภายใน Webhook",
"webhooks": "Webhooks"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Crafty Controller'a Hoşgeldiniz!"
},
"datatables": {
"loadingRecords": "Yükleniyor...",
"i18n": {
"aria": {
"sortAscending": ": artan sütun sıralamasını aktifleştir",
@ -673,4 +674,4 @@
"webhook_body": "Webhook Gövdesi",
"webhooks": "Webhooklar"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "Ласкаво просимо у Crafty Controller"
},
"datatables": {
"loadingRecords": "Завантаження...",
"i18n": {
"aria": {
"sortAscending": ": активуйте, щоб сортувати стовпці за зростанням",
@ -673,4 +674,4 @@
"webhook_body": "Код Вебхука",
"webhooks": "Вебхуки"
}
}
}

View File

@ -117,6 +117,7 @@
"welcome": "欢迎来到 Crafty Controller"
},
"datatables": {
"loadingRecords": "正在加载……",
"i18n": {
"aria": {
"sortAscending": ":激活对队列的升序排列",
@ -674,4 +675,4 @@
"webhook_body": "Webhook 消息体Body",
"webhooks": "Webhook"
}
}
}

View File

@ -17,6 +17,7 @@ from app.classes.models.users import HelperUsers
from app.classes.models.management import HelpersManagement
from app.classes.shared.import_helper import ImportHelpers
from app.classes.shared.websocket_manager import WebSocketManager
from app.classes.logging.log_formatter import JsonFormatter
console = Console()
helper = Helpers()
@ -284,6 +285,11 @@ def setup_logging(debug=True):
logging.config.dictConfig(logging_config)
# Apply JSON formatting to the "audit" handler
for handler in logging.getLogger().handlers:
if handler.name == "audit_log_handler":
handler.setFormatter(JsonFormatter())
else:
logging.basicConfig(level=logging.DEBUG)
logging.warning(f"Unable to read logging config from {logging_config_file}")