diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a4c5140..b147b200 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,13 +108,25 @@ foreach(file ${VANITY_FILES}) endforeach() # Move our migrations for MasterServer to run -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/) +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/dlu/) file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/dlu/*.sql) foreach(file ${SQL_FILES}) get_filename_component(file ${file} NAME) - if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/${file}) + if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/dlu/${file}) configure_file( - ${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/${file} + ${CMAKE_SOURCE_DIR}/migrations/dlu/${file} ${PROJECT_BINARY_DIR}/migrations/dlu/${file} + COPYONLY + ) + endif() +endforeach() + +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/migrations/cdserver/) +file(GLOB SQL_FILES ${CMAKE_SOURCE_DIR}/migrations/cdserver/*.sql) +foreach(file ${SQL_FILES}) + get_filename_component(file ${file} NAME) + if (NOT EXISTS ${PROJECT_BINARY_DIR}/migrations/cdserver/${file}) + configure_file( + ${CMAKE_SOURCE_DIR}/migrations/cdserver/${file} ${PROJECT_BINARY_DIR}/migrations/cdserver/${file} COPYONLY ) endif() diff --git a/README.md b/README.md index cdb1d343..8f9eaf8d 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,10 @@ certutil -hashfile SHA256 * Copy over or create symlinks from `locale.xml` in your client `locale` directory to the `build/locale` directory #### Client database -* Use `fdb_to_sqlite.py` in lcdr's utilities on `res/cdclient.fdb` in the unpacked client to convert the client database to `cdclient.sqlite` -* Move and rename `cdclient.sqlite` into `build/res/CDServer.sqlite` -* Run each SQL file in the order at which they appear [here](migrations/cdserver/) on the SQLite database +* Move the file `res/cdclient.fdb` from the unpacked client to the `build/res` folder on the server. +* The server will automatically copy and convert the file from fdb to sqlite should `CDServer.sqlite` not already exist. +* You can also convert the database manually using `fdb_to_sqlite.py` using lcdr's utilities. Just make sure to rename the file to `CDServer.sqlite` instead of `cdclient.sqlite`. +* Migrations to the database are automatically run on server start. When migrations are needed to be ran, the server may take a bit longer to start. ### Database Darkflame Universe utilizes a MySQL/MariaDB database for account and character information. @@ -229,7 +230,7 @@ Your build directory should now look like this: * **locale/** * locale.xml * **res/** - * CDServer.sqlite + * cdclient.fdb * chatplus_en_us.txt * **macros/** * ... diff --git a/dDatabase/CDClientDatabase.cpp b/dDatabase/CDClientDatabase.cpp index bf8485d6..4c2df1d2 100644 --- a/dDatabase/CDClientDatabase.cpp +++ b/dDatabase/CDClientDatabase.cpp @@ -14,6 +14,11 @@ CppSQLite3Query CDClientDatabase::ExecuteQuery(const std::string& query) { return conn->execQuery(query.c_str()); } +//! Updates the CDClient file with Data Manipulation Language (DML) commands. +int CDClientDatabase::ExecuteDML(const std::string& query) { + return conn->execDML(query.c_str()); +} + //! Makes prepared statements CppSQLite3Statement CDClientDatabase::CreatePreppedStmt(const std::string& query) { return conn->compileStatement(query.c_str()); diff --git a/dDatabase/CDClientDatabase.h b/dDatabase/CDClientDatabase.h index 6b254ebb..96f67d64 100644 --- a/dDatabase/CDClientDatabase.h +++ b/dDatabase/CDClientDatabase.h @@ -40,6 +40,14 @@ namespace CDClientDatabase { */ CppSQLite3Query ExecuteQuery(const std::string& query); + //! Updates the CDClient file with Data Manipulation Language (DML) commands. + /*! + \param query The DML command to run. DML command can be multiple queries in one string but only + the last one will return its number of updated rows. + \return The number of updated rows. + */ + int ExecuteDML(const std::string& query); + //! Queries the CDClient and parses arguments /*! \param query The query with formatted arguments diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index 1eb03887..017ebe32 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -1,11 +1,34 @@ #include "MigrationRunner.h" #include "BrickByBrickFix.h" +#include "CDClientDatabase.h" +#include "Database.h" +#include "Game.h" #include "GeneralUtils.h" +#include "dLogger.h" -#include -#include -#include +#include + +Migration LoadMigration(std::string path) { + Migration migration{}; + std::ifstream file("./migrations/" + path); + + if (file.is_open()) { + std::string line; + std::string total = ""; + + while (std::getline(file, line)) { + total += line; + } + + file.close(); + + migration.name = path; + migration.data = total; + } + + return migration; +} void MigrationRunner::RunMigrations() { auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());"); @@ -13,17 +36,14 @@ void MigrationRunner::RunMigrations() { delete stmt; sql::SQLString finalSQL = ""; - Migration checkMigration{}; bool runSd0Migrations = false; - for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/")) { - auto migration = LoadMigration(entry); + for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/dlu/")) { + auto migration = LoadMigration("dlu/" + entry); if (migration.data.empty()) { continue; } - checkMigration = migration; - stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;"); stmt->setString(1, migration.name); auto* res = stmt->executeQuery(); @@ -40,7 +60,7 @@ void MigrationRunner::RunMigrations() { } stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);"); - stmt->setString(1, entry); + stmt->setString(1, migration.name); stmt->execute(); delete stmt; } @@ -72,23 +92,39 @@ void MigrationRunner::RunMigrations() { } } -Migration MigrationRunner::LoadMigration(std::string path) { - Migration migration{}; - std::ifstream file("./migrations/" + path); +void MigrationRunner::RunSQLiteMigrations() { + auto* stmt = Database::CreatePreppedStmt("CREATE TABLE IF NOT EXISTS migration_history (name TEXT NOT NULL, date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP());"); + stmt->execute(); + delete stmt; - if (file.is_open()) { - std::string line; - std::string total = ""; + for (const auto& entry : GeneralUtils::GetFileNamesFromFolder("./migrations/cdserver/")) { + auto migration = LoadMigration("cdserver/" + entry); - while (std::getline(file, line)) { - total += line; + if (migration.data.empty()) continue; + + stmt = Database::CreatePreppedStmt("SELECT name FROM migration_history WHERE name = ?;"); + stmt->setString(1, migration.name); + auto* res = stmt->executeQuery(); + bool doExit = res->next(); + delete res; + delete stmt; + if (doExit) continue; + + // Doing these 1 migration at a time since one takes a long time and some may think it is crashing. + // This will at the least guarentee that the full migration needs to be run in order to be counted as "migrated". + Game::logger->Log("MigrationRunner", "Executing migration: %s. This may take a while. Do not shut down server.", migration.name.c_str()); + for (const auto& dml : GeneralUtils::SplitString(migration.data, ';')) { + if (dml.empty()) continue; + try { + CDClientDatabase::ExecuteDML(dml.c_str()); + } catch (CppSQLite3Exception& e) { + Game::logger->Log("MigrationRunner", "Encountered error running DML command: (%i) : %s", e.errorCode(), e.errorMessage()); + } } - - file.close(); - - migration.name = path; - migration.data = total; + stmt = Database::CreatePreppedStmt("INSERT INTO migration_history (name) VALUES (?);"); + stmt->setString(1, migration.name); + stmt->execute(); + delete stmt; } - - return migration; + Game::logger->Log("MigrationRunner", "CDServer database is up to date."); } diff --git a/dDatabase/MigrationRunner.h b/dDatabase/MigrationRunner.h index f5d1325a..0cb36d53 100644 --- a/dDatabase/MigrationRunner.h +++ b/dDatabase/MigrationRunner.h @@ -1,19 +1,13 @@ #pragma once -#include "Database.h" - -#include "dCommonVars.h" -#include "Game.h" -#include "dCommonVars.h" -#include "dLogger.h" +#include struct Migration { std::string data; std::string name; }; -class MigrationRunner { -public: - static void RunMigrations(); - static Migration LoadMigration(std::string path); +namespace MigrationRunner { + void RunMigrations(); + void RunSQLiteMigrations(); }; diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 538fb749..cd54913a 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -105,10 +105,34 @@ int main(int argc, char** argv) { const std::string cdclient_path = "./res/CDServer.sqlite"; std::ifstream cdclient_fd(cdclient_path); if (!cdclient_fd.good()) { - Game::logger->Log("WorldServer", "%s could not be opened", cdclient_path.c_str()); - return EXIT_FAILURE; + Game::logger->Log("WorldServer", "%s could not be opened. Looking for cdclient.fdb to convert to sqlite.", cdclient_path.c_str()); + cdclient_fd.close(); + + const std::string cdclientFdbPath = "./res/cdclient.fdb"; + cdclient_fd.open(cdclientFdbPath); + if (!cdclient_fd.good()) { + Game::logger->Log( + "WorldServer", "%s could not be opened." + "Please move a cdclient.fdb or an already converted database to build/res.", cdclientFdbPath.c_str()); + return EXIT_FAILURE; + } + Game::logger->Log("WorldServer", "Found %s. Clearing cdserver migration_history then copying and converting to sqlite.", cdclientFdbPath.c_str()); + auto stmt = Database::CreatePreppedStmt(R"#(DELETE FROM migration_history WHERE name LIKE "%cdserver%";)#"); + stmt->executeUpdate(); + delete stmt; + cdclient_fd.close(); + + std::string res = "python3 ../thirdparty/docker-utils/utils/fdb_to_sqlite.py " + cdclientFdbPath; + int r = system(res.c_str()); + if (r != 0) { + Game::logger->Log("MasterServer", "Failed to convert fdb to sqlite"); + return EXIT_FAILURE; + } + if (std::rename("./cdclient.sqlite", "./res/CDServer.sqlite") != 0) { + Game::logger->Log("MasterServer", "failed to move cdclient file."); + return EXIT_FAILURE; + } } - cdclient_fd.close(); //Connect to CDClient try { @@ -120,6 +144,9 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } + // Run migrations should any need to be run. + MigrationRunner::RunSQLiteMigrations(); + //Get CDClient initial information try { CDClientManager::Instance()->Initialize(); diff --git a/migrations/cdserver/0_nt_footrace.sql b/migrations/cdserver/0_nt_footrace.sql index fd37599e..0a40cfef 100644 --- a/migrations/cdserver/0_nt_footrace.sql +++ b/migrations/cdserver/0_nt_footrace.sql @@ -1,6 +1,2 @@ -BEGIN TRANSACTION; - UPDATE ComponentsRegistry SET component_id = 1901 WHERE id = 12916 AND component_type = 39; INSERT INTO ActivityRewards (objectTemplate, ActivityRewardIndex, activityRating, LootMatrixIndex, CurrencyIndex, ChallengeRating, description) VALUES (1901, 166, -1, 598, 1, 4, 'NT Foot Race'); - -COMMIT;