From 19d2c9ec1c7e3ce20bd60c5089e3b45a548c45d6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Sun, 10 Apr 2022 22:00:44 +0000 Subject: [PATCH 1/2] Revert "Merge branch 'bug/DB_Lock' into 'dev'" This reverts merge request !241 --- app/classes/models/crafty_permissions.py | 7 ++- app/classes/models/management.py | 41 ++++++++-------- app/classes/models/roles.py | 12 ++--- app/classes/models/server_permissions.py | 16 ++++--- app/classes/models/servers.py | 59 +++++++++++++----------- app/classes/models/users.py | 14 +++--- 6 files changed, 80 insertions(+), 69 deletions(-) diff --git a/app/classes/models/crafty_permissions.py b/app/classes/models/crafty_permissions.py index de43dbb9..665dad87 100644 --- a/app/classes/models/crafty_permissions.py +++ b/app/classes/models/crafty_permissions.py @@ -6,6 +6,7 @@ from app.classes.models.users import Users, ApiKeys, users_helper try: from peewee import ( + SqliteDatabase, Model, ForeignKeyField, CharField, @@ -13,7 +14,6 @@ try: DoesNotExist, ) from enum import Enum - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -21,9 +21,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) diff --git a/app/classes/models/management.py b/app/classes/models/management.py index 143133db..0d33e547 100644 --- a/app/classes/models/management.py +++ b/app/classes/models/management.py @@ -10,6 +10,7 @@ from app.classes.web.websocket_helper import websocket_helper try: from peewee import ( + SqliteDatabase, Model, ForeignKeyField, CharField, @@ -21,7 +22,6 @@ try: BooleanField, ) from playhouse.shortcuts import model_to_dict - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -29,9 +29,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) # ********************************************************************************** @@ -382,24 +381,28 @@ class helpers_management: conf["excluded_dirs"] = dirs_to_exclude conf["compress"] = compress if not new_row: - if backup_path is not None: - u1 = ( - Servers.update(backup_path=backup_path) - .where(Servers.server_id == server_id) - .execute() + with database.atomic(): + if backup_path is not None: + u1 = ( + Servers.update(backup_path=backup_path) + .where(Servers.server_id == server_id) + .execute() + ) + else: + u1 = 0 + u2 = ( + Backups.update(conf).where(Backups.server_id == server_id).execute() ) - else: - u1 = 0 - u2 = Backups.update(conf).where(Backups.server_id == server_id).execute() logger.debug(f"Updating existing backup record. {u1}+{u2} rows affected") else: - conf["server_id"] = server_id - if backup_path is not None: - Servers.update(backup_path=backup_path).where( - Servers.server_id == server_id - ) - Backups.create(**conf) - logger.debug("Creating new backup record.") + with database.atomic(): + conf["server_id"] = server_id + if backup_path is not None: + Servers.update(backup_path=backup_path).where( + Servers.server_id == server_id + ) + Backups.create(**conf) + logger.debug("Creating new backup record.") def get_excluded_backup_dirs(self, server_id: int): excluded_dirs = self.get_backup_config(server_id)["excluded_dirs"] diff --git a/app/classes/models/roles.py b/app/classes/models/roles.py index 82fdfc26..ff705e96 100644 --- a/app/classes/models/roles.py +++ b/app/classes/models/roles.py @@ -5,6 +5,7 @@ from app.classes.shared.helpers import helper try: from peewee import ( + SqliteDatabase, Model, CharField, DoesNotExist, @@ -12,7 +13,6 @@ try: DateTimeField, ) from playhouse.shortcuts import model_to_dict - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -20,9 +20,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) # ********************************************************************************** @@ -75,8 +74,9 @@ class helper_roles: @staticmethod def remove_role(role_id): - role = Roles.get(Roles.role_id == role_id) - return role.delete_instance() + with database.atomic(): + role = Roles.get(Roles.role_id == role_id) + return role.delete_instance() @staticmethod def role_id_exists(role_id): diff --git a/app/classes/models/server_permissions.py b/app/classes/models/server_permissions.py index 1d0a5d35..12f42b2f 100644 --- a/app/classes/models/server_permissions.py +++ b/app/classes/models/server_permissions.py @@ -8,6 +8,7 @@ from app.classes.shared.permission_helper import permission_helper try: from peewee import ( + SqliteDatabase, Model, ForeignKeyField, CharField, @@ -15,7 +16,6 @@ try: JOIN, ) from enum import Enum - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -23,9 +23,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) # ********************************************************************************** @@ -194,9 +193,12 @@ class Permissions_Servers: @staticmethod def remove_roles_of_server(server_id): - return ( - Role_Servers.delete().where(Role_Servers.server_id == server_id).execute() - ) + with database.atomic(): + return ( + Role_Servers.delete() + .where(Role_Servers.server_id == server_id) + .execute() + ) @staticmethod def get_user_id_permissions_mask(user_id, server_id: str): diff --git a/app/classes/models/servers.py b/app/classes/models/servers.py index af2eea7a..f00704fe 100644 --- a/app/classes/models/servers.py +++ b/app/classes/models/servers.py @@ -6,6 +6,7 @@ from app.classes.shared.main_models import db_helper try: from peewee import ( + SqliteDatabase, Model, ForeignKeyField, CharField, @@ -15,7 +16,6 @@ try: IntegerField, FloatField, ) - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -23,9 +23,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) # ********************************************************************************** @@ -142,7 +141,8 @@ class helper_servers: @staticmethod def remove_server(server_id): - Servers.delete().where(Servers.server_id == server_id).execute() + with database.atomic(): + Servers.delete().where(Servers.server_id == server_id).execute() @staticmethod def get_server_data_by_id(server_id): @@ -224,21 +224,24 @@ class helper_servers: @staticmethod def sever_crashed(server_id): - Server_Stats.update(crashed=True).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(crashed=True).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def set_download(server_id): - Server_Stats.update(downloading=True).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(downloading=True).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def finish_download(server_id): - Server_Stats.update(downloading=False).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(downloading=False).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def get_download_status(server_id): @@ -249,9 +252,10 @@ class helper_servers: @staticmethod def server_crash_reset(server_id): - Server_Stats.update(crashed=False).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(crashed=False).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def is_crashed(server_id): @@ -268,9 +272,10 @@ class helper_servers: Server_Stats.select().where(Server_Stats.server_id == server_id) except Exception as ex: logger.error(f"Database entry not found! {ex}") - Server_Stats.update(updating=value).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(updating=value).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def get_update_status(server_id): @@ -288,9 +293,10 @@ class helper_servers: except Exception as ex: logger.error(f"Database entry not found! {ex}") return - Server_Stats.update(first_run=False).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(first_run=False).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def get_first_run(server_id): @@ -331,9 +337,10 @@ class helper_servers: Server_Stats.select().where(Server_Stats.server_id == server_id) except Exception as ex: logger.error(f"Database entry not found! {ex}") - Server_Stats.update(waiting_start=value).where( - Server_Stats.server_id == server_id - ).execute() + with database.atomic(): + Server_Stats.update(waiting_start=value).where( + Server_Stats.server_id == server_id + ).execute() @staticmethod def get_waiting_start(server_id): diff --git a/app/classes/models/users.py b/app/classes/models/users.py index 1ac3c74d..f9e9f311 100644 --- a/app/classes/models/users.py +++ b/app/classes/models/users.py @@ -7,6 +7,7 @@ from app.classes.shared.helpers import helper try: from peewee import ( + SqliteDatabase, Model, ForeignKeyField, CharField, @@ -18,7 +19,6 @@ try: JOIN, ) from playhouse.shortcuts import model_to_dict - from playhouse.sqliteq import SqliteQueueDatabase except ModuleNotFoundError as e: helper.auto_installer_fix(e) @@ -26,9 +26,8 @@ except ModuleNotFoundError as e: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( - helper.db_path - # pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} +database = SqliteDatabase( + helper.db_path, pragmas={"journal_mode": "wal", "cache_size": -1024 * 10} ) # ********************************************************************************** @@ -233,9 +232,10 @@ class helper_users: @staticmethod def remove_user(user_id): - User_Roles.delete().where(User_Roles.user_id == user_id).execute() - user = Users.get(Users.user_id == user_id) - return user.delete_instance() + with database.atomic(): + User_Roles.delete().where(User_Roles.user_id == user_id).execute() + user = Users.get(Users.user_id == user_id) + return user.delete_instance() @staticmethod def set_support_path(user_id, support_path): From 711d41533510847d4c56174a2baf5e7ea338f809 Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Sun, 10 Apr 2022 18:04:16 -0400 Subject: [PATCH 2/2] Remove queue --- app/classes/shared/main_models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/classes/shared/main_models.py b/app/classes/shared/main_models.py index 155892a7..8079f7d9 100644 --- a/app/classes/shared/main_models.py +++ b/app/classes/shared/main_models.py @@ -10,8 +10,7 @@ Users = Users try: # pylint: disable=unused-import - from peewee import fn - from playhouse.sqliteq import SqliteQueueDatabase + from peewee import SqliteDatabase, fn from playhouse.shortcuts import model_to_dict except ModuleNotFoundError as err: @@ -20,7 +19,7 @@ except ModuleNotFoundError as err: logger = logging.getLogger(__name__) peewee_logger = logging.getLogger("peewee") peewee_logger.setLevel(logging.INFO) -database = SqliteQueueDatabase( +database = SqliteDatabase( helper.db_path # This is commented out after presenting issues when # moving from SQLiteDatabase to SqliteQueueDatabase