Merge pull request #887 from EmosewaMC/ServerShutdown

Improve server shutdown process

dAuthServer/AuthServer.cpp

@@ -25,6 +25,7 @@ namespace Game {
 	dLogger* logger = nullptr;
 	dServer* server = nullptr;
 	dConfig* config = nullptr;
+	bool shouldShutdown = false;
 }
 
 dLogger* SetupLogger();
@@ -83,7 +84,7 @@ int main(int argc, char** argv) {
 	if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
 	if (Game::config->GetValue("port") != "") ourPort = std::atoi(Game::config->GetValue("port").c_str());
 
-	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Auth, Game::config);
+	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Auth, Game::config, &Game::shouldShutdown);
 
 	//Run it until server gets a kill message from Master:
 	auto t = std::chrono::high_resolution_clock::now();
@@ -92,7 +93,7 @@ int main(int argc, char** argv) {
 	int framesSinceMasterDisconnect = 0;
 	int framesSinceLastSQLPing = 0;
 
-	while (true) {
+	while (!Game::shouldShutdown) {
 		//Check if we're still connected to master:
 		if (!Game::server->GetIsConnectedToMaster()) {
 			framesSinceMasterDisconnect++;

dChatServer/ChatServer.cpp

@@ -25,6 +25,7 @@ namespace Game {
 	dConfig* config = nullptr;
 	dChatFilter* chatFilter = nullptr;
 	AssetManager* assetManager = nullptr;
+	bool shouldShutdown = false;
 }
 
 //RakNet includes:
@@ -103,7 +104,7 @@ int main(int argc, char** argv) {
 	if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
 	if (Game::config->GetValue("port") != "") ourPort = std::atoi(Game::config->GetValue("port").c_str());
 
-	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat, Game::config);
+	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::Chat, Game::config, &Game::shouldShutdown);
 
 	Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(Game::config->GetValue("dont_generate_dcf"))));
 
@@ -114,7 +115,7 @@ int main(int argc, char** argv) {
 	int framesSinceMasterDisconnect = 0;
 	int framesSinceLastSQLPing = 0;
 
-	while (true) {
+	while (!Game::shouldShutdown) {
 		//Check if we're still connected to master:
 		if (!Game::server->GetIsConnectedToMaster()) {
 			framesSinceMasterDisconnect++;

dCommon/Game.h

@@ -21,4 +21,5 @@ namespace Game {
 	extern RakPeerInterface* chatServer;
 	extern AssetManager* assetManager;
 	extern SystemAddress chatSysAddr;
+	extern bool shouldShutdown;
 }

dMasterServer/InstanceManager.cpp

@@ -150,7 +150,7 @@ void InstanceManager::RemoveInstance(Instance* instance) {
 		if (m_Instances[i] == instance) {
 			instance->SetShutdownComplete(true);
 
-			RedirectPendingRequests(instance);
+			if (!Game::shouldShutdown) RedirectPendingRequests(instance);
 
 			delete m_Instances[i];
 
@@ -391,5 +391,5 @@ void Instance::Shutdown() {
 
 	Game::server->Send(&bitStream, this->m_SysAddr, false);
 
-	Game::logger->Log("Instance", "Triggered world shutdown");
+	Game::logger->Log("Instance", "Triggered world shutdown for zone/clone/instance %i/%i/%i", GetMapID(), GetCloneID(), GetInstanceID());
 }

dMasterServer/MasterServer.cpp

@@ -48,17 +48,18 @@ namespace Game {
 	InstanceManager* im = nullptr;
 	dConfig* config = nullptr;
 	AssetManager* assetManager = nullptr;
+	bool shouldShutdown = false;
 } //namespace Game
 
 bool shutdownSequenceStarted = false;
-void ShutdownSequence();
+void ShutdownSequence(int signal = -1);
-int FinalizeShutdown();
+int FinalizeShutdown(int signal = -1);
 dLogger* SetupLogger();
 void StartAuthServer();
 void StartChatServer();
 void HandlePacket(Packet* packet);
 std::map<uint32_t, std::string> activeSessions;
-bool shouldShutdown = false;
+SystemAddress authServerMasterPeerSysAddr;
 SystemAddress chatServerMasterPeerSysAddr;
 
 int main(int argc, char** argv) {
@@ -71,9 +72,9 @@ int main(int argc, char** argv) {
 #endif
 
 	//Triggers the shutdown sequence at application exit
-	std::atexit(ShutdownSequence);
+	std::atexit([]() { ShutdownSequence(); });
-	signal(SIGINT, [](int) { ShutdownSequence(); });
+	signal(SIGINT, [](int signal) { ShutdownSequence(EXIT_FAILURE); });
-	signal(SIGTERM, [](int) { ShutdownSequence(); });
+	signal(SIGTERM, [](int signal) { ShutdownSequence(EXIT_FAILURE); });
 
 	//Create all the objects we need to run our service:
 	Game::logger = SetupLogger();
@@ -241,7 +242,7 @@ int main(int argc, char** argv) {
 	if (Game::config->GetValue("max_clients") != "") maxClients = std::stoi(Game::config->GetValue("max_clients"));
 	if (Game::config->GetValue("port") != "") ourPort = std::stoi(Game::config->GetValue("port"));
 
-	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, true, false, Game::logger, "", 0, ServerType::Master, Game::config);
+	Game::server = new dServer(Game::config->GetValue("external_ip"), ourPort, 0, maxClients, true, false, Game::logger, "", 0, ServerType::Master, Game::config, &Game::shouldShutdown);
 
 	//Query for the database for a server labeled "master"
 	auto* masterLookupStatement = Database::CreatePreppedStmt("SELECT id FROM `servers` WHERE `name` = 'master'");
@@ -278,8 +279,8 @@ int main(int argc, char** argv) {
 	if (Game::config->GetValue("prestart_servers") != "" && Game::config->GetValue("prestart_servers") == "1") {
 		StartChatServer();
 
-		Game::im->GetInstance(0, false, 0)->SetIsReady(true);
+		Game::im->GetInstance(0, false, 0);
-		Game::im->GetInstance(1000, false, 0)->SetIsReady(true);
+		Game::im->GetInstance(1000, false, 0);
 
 		StartAuthServer();
 	}
@@ -328,7 +329,7 @@ int main(int argc, char** argv) {
 			framesSinceLastSQLPing++;
 
 		//10m shutdown for universe kill command
-		if (shouldShutdown) {
+		if (Game::shouldShutdown) {
 			if (framesSinceKillUniverseCommand >= 40000) {
 				//Break main loop and exit
 				break;
@@ -375,9 +376,7 @@ int main(int argc, char** argv) {
 		t += std::chrono::milliseconds(highFrameRate);
 		std::this_thread::sleep_until(t);
 	}
-	FinalizeShutdown();
+	return FinalizeShutdown(EXIT_SUCCESS);
-	exit(EXIT_SUCCESS);
-	return EXIT_SUCCESS;
 }
 
 dLogger* SetupLogger() {
@@ -406,9 +405,15 @@ void HandlePacket(Packet* packet) {
 			Game::im->RemoveInstance(instance); //Delete the old
 		}
 
-		if (packet->systemAddress == chatServerMasterPeerSysAddr && !shouldShutdown) {
+		if (packet->systemAddress == chatServerMasterPeerSysAddr) {
+			chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
 			StartChatServer();
 		}
+
+		if (packet->systemAddress == authServerMasterPeerSysAddr) {
+			authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
+			StartAuthServer();
+		}
 	}
 
 	if (packet->data[0] == ID_CONNECTION_LOST) {
@@ -422,9 +427,15 @@ void HandlePacket(Packet* packet) {
 			//Game::im->GetInstance(zoneID.GetMapID(), false, 0); //Create the new
 		}
 
-		if (packet->systemAddress == chatServerMasterPeerSysAddr && !shouldShutdown) {
+		if (packet->systemAddress == chatServerMasterPeerSysAddr) {
+			chatServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
 			StartChatServer();
 		}
+
+		if (packet->systemAddress == authServerMasterPeerSysAddr) {
+			authServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
+			StartAuthServer();
+		}
 	}
 
 	if (packet->data[1] == MASTER) {
@@ -520,6 +531,14 @@ void HandlePacket(Packet* packet) {
 				chatServerMasterPeerSysAddr = copy;
 			}
 
+			if (theirServerType == ServerType::Auth) {
+				SystemAddress copy;
+				copy.binaryAddress = packet->systemAddress.binaryAddress;
+				copy.port = packet->systemAddress.port;
+
+				authServerMasterPeerSysAddr = copy;
+			}
+
 			Game::logger->Log("MasterServer", "Received server info, instance: %i port: %i", theirInstanceID, theirPort);
 
 			break;
@@ -744,7 +763,7 @@ void HandlePacket(Packet* packet) {
 
 		case MSG_MASTER_SHUTDOWN_UNIVERSE: {
 			Game::logger->Log("MasterServer", "Received shutdown universe command, shutting down in 10 minutes.");
-			shouldShutdown = true;
+			Game::shouldShutdown = true;
 			break;
 		}
 
@@ -755,6 +774,10 @@ void HandlePacket(Packet* packet) {
 	}
 
 void StartChatServer() {
+	if (Game::shouldShutdown) {
+		Game::logger->Log("MasterServer", "Currently shutting down.  Chat will not be restarted.");
+		return;
+	}
 #ifdef __APPLE__
 	//macOS doesn't need sudo to run on ports < 1024
 	auto result = system(((BinaryPathFinder::GetBinaryDir() / "ChatServer").string() + "&").c_str());
@@ -770,6 +793,10 @@ void StartChatServer() {
 	}
 
 void StartAuthServer() {
+	if (Game::shouldShutdown) {
+		Game::logger->Log("MasterServer", "Currently shutting down.  Auth will not be restarted.");
+		return;
+	}
 #ifdef __APPLE__
 	auto result = system(((BinaryPathFinder::GetBinaryDir() / "AuthServer").string() + "&").c_str());
 #elif _WIN32
@@ -783,21 +810,19 @@ void StartAuthServer() {
 #endif
 }
 
-void ShutdownSequence() {
+void ShutdownSequence(int signal) {
 	if (shutdownSequenceStarted) {
 		return;
 	}
 
 	shutdownSequenceStarted = true;
+	Game::shouldShutdown = true;
 
-	if (Game::im) {
+	{
-		for (auto* instance : Game::im->GetInstances()) {
+		CBITSTREAM;
-			if (instance == nullptr) {
+		PacketUtils::WriteHeader(bitStream, MASTER, MSG_MASTER_SHUTDOWN);
-				continue;
+		Game::server->Send(&bitStream, UNASSIGNED_SYSTEM_ADDRESS, true);
-			}
+		Game::logger->Log("MasterServer", "Triggered master shutdown");
-
-			instance->Shutdown();
-		}
 	}
 
 	auto* objIdManager = ObjectIDManager::TryInstance();
@@ -810,13 +835,23 @@ void ShutdownSequence() {
 	auto ticks = 0;
 
 	if (!Game::im) {
-		exit(EXIT_SUCCESS);
+		FinalizeShutdown(EXIT_FAILURE);
+	}
+
+	// A server might not be finished spinning up yet, remove all of those here.
+	for (auto instance : Game::im->GetInstances()) {
+		if (!instance->GetIsReady()) {
+			Game::im->RemoveInstance(instance);
+		}
+	}
+
+	for (auto instance : Game::im->GetInstances()) {
+		instance->SetIsShuttingDown(true);
 	}
 
 	Game::logger->Log("MasterServer", "Attempting to shutdown instances, max 60 seconds...");
 
 	while (true) {
-
 		auto packet = Game::server->Receive();
 		if (packet) {
 			HandlePacket(packet);
@@ -836,7 +871,7 @@ void ShutdownSequence() {
 			}
 		}
  
-		if (done) {
+		if (done && authServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS && chatServerMasterPeerSysAddr == UNASSIGNED_SYSTEM_ADDRESS) {
 			Game::logger->Log("MasterServer", "Finished shutting down MasterServer!");
 			break;
 		}
@@ -852,10 +887,10 @@ void ShutdownSequence() {
 		}
 	}
 
-	FinalizeShutdown();
+	FinalizeShutdown(signal);
 }
 
-int FinalizeShutdown() {
+int FinalizeShutdown(int signal) {
 	//Delete our objects here:
 	Database::Destroy("MasterServer");
 	if (Game::config) delete Game::config;
@@ -863,6 +898,6 @@ int FinalizeShutdown() {
 	if (Game::server) delete Game::server;
 	if (Game::logger) delete Game::logger;
 
-	exit(EXIT_SUCCESS);
+	if (signal != EXIT_SUCCESS) exit(signal);
-	return EXIT_SUCCESS;
+	return signal;
 }

dNet/dServer.cpp

@@ -36,7 +36,7 @@ public:
 	}
 } ReceiveDownloadCompleteCB;
 
-dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, unsigned int zoneID) {
+dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, bool* shouldShutdown, unsigned int zoneID) {
 	mIP = ip;
 	mPort = port;
 	mZoneID = zoneID;
@@ -52,6 +52,7 @@ dServer::dServer(const std::string& ip, int port, int instanceID, int maxConnect
 	mReplicaManager = nullptr;
 	mServerType = serverType;
 	mConfig = config;
+	mShouldShutdown = shouldShutdown;
 	//Attempt to start our server here:
 	mIsOkay = Startup();
 
@@ -124,6 +125,9 @@ Packet* dServer::ReceiveFromMaster() {
 					ZoneInstanceManager::Instance()->HandleRequestZoneTransferResponse(requestID, packet);
 					break;
 				}
+				case MSG_MASTER_SHUTDOWN:
+					*mShouldShutdown = true;
+					break;
 
 				//When we handle these packets in World instead dServer, we just return the packet's pointer.
 				default:

dNet/dServer.h

@@ -18,7 +18,20 @@ class dServer {
 public:
 	// Default constructor should only used for testing!
 	dServer() {};
-	dServer(const std::string& ip, int port, int instanceID, int maxConnections, bool isInternal, bool useEncryption, dLogger* logger, const std::string masterIP, int masterPort, ServerType serverType, dConfig* config, unsigned int zoneID = 0);
+	dServer(
+		const std::string& ip,
+		int port,
+		int instanceID,
+		int maxConnections,
+		bool isInternal,
+		bool useEncryption,
+		dLogger* logger,
+		const std::string masterIP,
+		int masterPort,
+		ServerType serverType,
+		dConfig* config,
+		bool* shouldShutdown,
+		unsigned int zoneID = 0);
 	~dServer();
 
 	Packet* ReceiveFromMaster();
@@ -64,6 +77,11 @@ private:
 	RakPeerInterface* mPeer = nullptr;
 	ReplicaManager* mReplicaManager = nullptr;
 	NetworkIDManager* mNetIDManager = nullptr;
+
+	/**
+	 * Whether or not to shut down the server.  Pointer to Game::shouldShutdown.
+	 */
+	bool* mShouldShutdown = nullptr;
 	SocketDescriptor mSocketDescriptor;
 	std::string mIP;
 	int mPort;

dWorldServer/WorldServer.cpp

@@ -70,11 +70,11 @@ namespace Game {
 	RakPeerInterface* chatServer = nullptr;
 	std::mt19937 randomEngine;
 	SystemAddress chatSysAddr;
-}
+	bool shouldShutdown = false;
+} // namespace Game
 
 bool chatDisabled = false;
 bool chatConnected = false;
-bool worldShutdownSequenceStarted = false;
 bool worldShutdownSequenceComplete = false;
 void WorldShutdownSequence();
 void WorldShutdownProcess(uint32_t zoneId);
@@ -200,7 +200,7 @@ int main(int argc, char** argv) {
 	LootGenerator::Instance();
 	Game::chatFilter = new dChatFilter(Game::assetManager->GetResPath().string() + "/chatplus_en_us", bool(std::stoi(Game::config->GetValue("dont_generate_dcf"))));
 
-	Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, Game::config, zoneID);
+	Game::server = new dServer(masterIP, ourPort, instanceID, maxClients, false, true, Game::logger, masterIP, masterPort, ServerType::World, Game::config, &Game::shouldShutdown, zoneID);
 
 	//Connect to the chat server:
 	int chatPort = 1501;
@@ -315,9 +315,9 @@ int main(int argc, char** argv) {
 			framesSinceMasterDisconnect++;
 
 			int framesToWaitForMaster = ready ? 10 : 200;
-			if (framesSinceMasterDisconnect >= framesToWaitForMaster && !worldShutdownSequenceStarted) {
+			if (framesSinceMasterDisconnect >= framesToWaitForMaster && !Game::shouldShutdown) {
 				Game::logger->Log("WorldServer", "Game loop running but no connection to master for %d frames, shutting down", framesToWaitForMaster);
-				worldShutdownSequenceStarted = true;
+				Game::shouldShutdown = true;
 			}
 		} else framesSinceMasterDisconnect = 0;
 
@@ -413,7 +413,7 @@ int main(int argc, char** argv) {
 
 			//If we haven't had any players for a while, time out and shut down:
 			if (framesSinceLastUser == (cloneID != 0 ? 4000 : 40000)) {
-				worldShutdownSequenceStarted = true;
+				Game::shouldShutdown = true;
 			}
 		} else {
 			framesSinceLastUser = 0;
@@ -470,7 +470,7 @@ int main(int argc, char** argv) {
 			}
 		}
 
-		if (worldShutdownSequenceStarted && !worldShutdownSequenceComplete) {
+		if (Game::shouldShutdown && !worldShutdownSequenceComplete) {
 			WorldShutdownProcess(zoneID);
 			break;
 		}
@@ -789,7 +789,7 @@ void HandlePacket(Packet* packet) {
 		}
 
 		case MSG_MASTER_SHUTDOWN: {
-			worldShutdownSequenceStarted = true;
+			Game::shouldShutdown = true;
 			Game::logger->Log("WorldServer", "Got shutdown request from master, zone (%i), instance (%i)", Game::server->GetZoneID(), Game::server->GetInstanceID());
 			break;
 		}
@@ -1259,11 +1259,11 @@ void WorldShutdownProcess(uint32_t zoneId) {
 }
 
 void WorldShutdownSequence() {
-	if (worldShutdownSequenceStarted || worldShutdownSequenceComplete) {
+	if (Game::shouldShutdown || worldShutdownSequenceComplete) {
 		return;
 	}
 
-	worldShutdownSequenceStarted = true;
+	Game::shouldShutdown = true;
 
 	Game::logger->Log("WorldServer", "Zone (%i) instance (%i) shutting down outside of main loop!", Game::server->GetZoneID(), instanceID);
 	WorldShutdownProcess(Game::server->GetZoneID());