From 906887bda9d2f0d0b115ba07b10bc27b3a1e6863 Mon Sep 17 00:00:00 2001 From: David Markowitz <39972741+EmosewaMC@users.noreply.github.com> Date: Sun, 30 Oct 2022 00:38:43 -0700 Subject: [PATCH] Add automatic cdclient migration runner support and setup (#789) * Add automatic migrations for CDServer Add support to automatically migrate and update CDServers with new migrations. Also adds support to simplify the setup process by simply putting the fdb in the res folder and letting the server convert it to sqlite. This reduces the amount of back and forth when setting up a server. * Remove transaction language * Add DML execution `poggers` Add a way to execute DML commands through the sqlite connection on the server. * Make DML Commands more robust On the off chance the server is shutdown before the whole migration is run, lets just not add it to our "finished list" until the whole file is done. * Update README --- CMakeLists.txt | 18 +++++- README.md | 9 +-- dDatabase/CDClientDatabase.cpp | 5 ++ dDatabase/CDClientDatabase.h | 8 +++ dDatabase/MigrationRunner.cpp | 84 +++++++++++++++++++-------- dDatabase/MigrationRunner.h | 14 ++--- dMasterServer/MasterServer.cpp | 33 ++++++++++- migrations/cdserver/0_nt_footrace.sql | 4 -- 8 files changed, 127 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a4c5140..b147b200 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,13 +108,25 @@ foreach(file ${VANITY_FILES}) endforeach() # Move our migrations for MasterServer to run -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/) +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/) file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql) foreach(file ${SQL_FILES}) get_filename_component(file ${file} NAME) - if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/${file}) + if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/dlu/${file}) configure_file( - ${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/${file} + ${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file} + COPYONLY + ) + endif() +endforeach() + +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/) +file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql) +foreach(file ${SQL_FILES}) + get_filename_component(file ${file} NAME) + if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/cdserver/${file}) + configure_file( + ${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file} COPYONLY ) endif() diff --git a/README.md b/README.md index cdb1d343..8f9eaf8d 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,10 @@ certutil -hashfile SHA256 * Copy over or create symlinks from `locale.xml` in your client `locale` directory to the `build/locale` directory #### Client database -* Use `fdb_to_sqlite.py` in lcdr's utilities on `res/cdclient.fdb` in the unpacked client to convert the client database to `cdclient.sqlite` -* Move and rename `cdclient.sqlite` into `build/res/CDServer.sqlite` -* Run each SQL file in the order at which they appear [here](migrations/cdserver/) on the SQLite database +* Move the file `res/cdclient.fdb` from the unpacked client to the `build/res` folder on the server. +* The server will automatically copy and convert the file from fdb to sqlite should `CDServer.sqlite` not already exist. +* You can also convert the database manually using `fdb_to_sqlite.py` using lcdr's utilities. Just make sure to rename the file to `CDServer.sqlite` instead of `cdclient.sqlite`. +* Migrations to the database are automatically run on server start. When migrations are needed to be ran, the server may take a bit longer to start. ### Database Darkflame Universe utilizes a MySQL/MariaDB database for account and character information. @@ -229,7 +230,7 @@ Your build directory should now look like this: * **locale/** * locale.xml * **res/** - * CDServer.sqlite + * cdclient.fdb * chatplus_en_us.txt * **macros/** * ... diff --git a/dDatabase/CDClientDatabase.cpp b/dDatabase/CDClientDatabase.cpp index bf8485d6..4c2df1d2 100644 --- a/dDatabase/CDClientDatabase.cpp +++ b/dDatabase/CDClientDatabase.cpp @@ -14,6 +14,11 @@ CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) { return conn->execQuery(query.c_str()); } +//! Updates the CDClient file with Data Manipulation Language (DML) commands. +int CDClientDatabase::ExecuteDML(const std::string& query) { + return conn->execDML(query.c_str()); +} + //! Makes prepared statements CppSQLite3Statement CDClientDatabase::CreatePreppedStmt(const std::string& query) { return conn->compileStatement(query.c_str()); diff --git a/dDatabase/CDClientDatabase.h b/dDatabase/CDClientDatabase.h index 6b254ebb..96f67d64 100644 --- a/dDatabase/CDClientDatabase.h +++ b/dDatabase/CDClientDatabase.h @@ -40,6 +40,14 @@ namespace CDClientDatabase { */ CppSQLite3Query ExecuteQuery(const std::string& query); + //! Updates the CDClient file with Data Manipulation Language (DML) commands. + /*! + \param query The DML command to run. DML command can be multiple queries in one string but only + the last one will return its number of updated rows. + \return The number of updated rows. + */ + int ExecuteDML(const std::string& query); + //! Queries the CDClient and parses arguments /*! \param query The query with formatted arguments diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index 1eb03887..017ebe32 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -1,11 +1,34 @@ #include "MigrationRunner.h" #include "BrickByBrickFix.h" +#include "CDClientDatabase.h" +#include "Database.h" +#include "Game.h" #include "GeneralUtils.h" +#include "dLogger.h" -#include -#include -#include +#include + +Migration LoadMigration(std::string path) { + Migration migration{}; + std::ifstream file("./migrations/" + path); + + if (file.is_open()) { + std::string line; + std::string total = ""; + + while (std::getline(file, line)) { + total += line; + } + + file.close(); + + migration.name = path; + migration.data = total; + } + + return migration; +} void MigrationRunner::RunMigrations() { auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());"); @@ -13,17 +36,14 @@ void MigrationRunner::RunMigrations() { delete stmt; sql::SQLString finalSQL = ""; - Migration checkMigration{}; bool runSd0Migrations = false; - for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/")) { - auto migration = LoadMigration(entry); + for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/dlu/")) { + auto migration = LoadMigration("dlu/" + entry); if (migration.data.empty()) { continue; } - checkMigration = migration; - stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;"); stmt->setString(1, migration.name); auto* res = stmt->executeQuery(); @@ -40,7 +60,7 @@ void MigrationRunner::RunMigrations() { } stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);"); - stmt->setString(1, entry); + stmt->setString(1, migration.name); stmt->execute(); delete stmt; } @@ -72,23 +92,39 @@ void MigrationRunner::RunMigrations() { } } -Migration MigrationRunner::LoadMigration(std::string path) { - Migration migration{}; - std::ifstream file("./migrations/" + path); +void MigrationRunner::RunSQLiteMigrations() { + auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());"); + stmt->execute(); + delete stmt; - if (file.is_open()) { - std::string line; - std::string total = ""; + for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/cdserver/")) { + auto migration = LoadMigration("cdserver/" + entry); - while (std::getline(file, line)) { - total += line; + if (migration.data.empty()) continue; + + stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;"); + stmt->setString(1, migration.name); + auto* res = stmt->executeQuery(); + bool doExit = res->next(); + delete res; + delete stmt; + if (doExit) continue; + + // Doing these 1 migration at a time since one takes a long time and some may think it is crashing. + // This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated". + Game::logger->Log("MigrationRunner", "Executing migration: %s. This may take a while. Do not shut down server.", migration.name.c_str()); + for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) { + if (dml.empty()) continue; + try { + CDClientDatabase::ExecuteDML(dml.c_str()); + } catch (CppSQLite3Exception& e) { + Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage()); + } } - - file.close(); - - migration.name = path; - migration.data = total; + stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);"); + stmt->setString(1, migration.name); + stmt->execute(); + delete stmt; } - - return migration; + Game::logger->Log("MigrationRunner", "CDServer database is up to date."); } diff --git a/dDatabase/MigrationRunner.h b/dDatabase/MigrationRunner.h index f5d1325a..0cb36d53 100644 --- a/dDatabase/MigrationRunner.h +++ b/dDatabase/MigrationRunner.h @@ -1,19 +1,13 @@ #pragma once -#include "Database.h" - -#include "dCommonVars.h" -#include "Game.h" -#include "dCommonVars.h" -#include "dLogger.h" +#include struct Migration { std::string data; std::string name; }; -class MigrationRunner { -public: - static void RunMigrations(); - static Migration LoadMigration(std::string path); +namespace MigrationRunner { + void RunMigrations(); + void RunSQLiteMigrations(); }; diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 538fb749..cd54913a 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -105,10 +105,34 @@ int main(int argc, char** argv) { const std::string cdclient_path = "./res/CDServer.sqlite"; std::ifstream cdclient_fd(cdclient_path); if (!cdclient_fd.good()) { - Game::logger->Log("WorldServer", "%s could not be opened", cdclient_path.c_str()); - return EXIT_FAILURE; + Game::logger->Log("WorldServer", "%s could not be opened. Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str()); + cdclient_fd.close(); + + const std::string cdclientFdbPath = "./res/cdclient.fdb"; + cdclient_fd.open(cdclientFdbPath); + if (!cdclient_fd.good()) { + Game::logger->Log( + "WorldServer", "%s could not be opened." + "Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str()); + return EXIT_FAILURE; + } + Game::logger->Log("WorldServer", "Found %s. Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str()); + auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#"); + stmt->executeUpdate(); + delete stmt; + cdclient_fd.close(); + + std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath; + int r = system(res.c_str()); + if (r != 0) { + Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite"); + return EXIT_FAILURE; + } + if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) { + Game::logger->Log("MasterServer", "failed to move cdclient file."); + return EXIT_FAILURE; + } } - cdclient_fd.close(); //Connect to CDClient try { @@ -120,6 +144,9 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } + // Run migrations should any need to be run. + MigrationRunner::RunSQLiteMigrations(); + //Get CDClient initial information try { CDClientManager::Instance()->Initialize(); diff --git a/migrations/cdserver/0_nt_footrace.sql b/migrations/cdserver/0_nt_footrace.sql index fd37599e..0a40cfef 100644 --- a/migrations/cdserver/0_nt_footrace.sql +++ b/migrations/cdserver/0_nt_footrace.sql @@ -1,6 +1,2 @@ -BEGIN TRANSACTION; - UPDATE ComponentsRegistry SET component_id = 1901 WHERE id = 12916 AND component_type = 39; INSERT INTO ActivityRewards (objectTemplate, ActivityRewardIndex, activityRating, LootMatrixIndex, CurrencyIndex, ChallengeRating, description) VALUES (1901, 166, -1, 598, 1, 4, 'NT Foot Race'); - -COMMIT;