Merge pull request #90 from TheNoim/main

Completely automated docker setup

.dockerignore

@@ -0,0 +1,10 @@
+.git
+Dockerfile
+*.md
+logo.png
+versions.txt
+build.sh
+docker-compose.yml
+.env
+docker/__pycache__
+.env.example

.env.example

@@ -0,0 +1,17 @@
+# Full path to the LEGO Universe client
+CLIENT_PATH=/Users/someuser/LEGO Universe
+# Can improve build time
+BUILD_THREADS=1
+# Updates NET_VERSION in CMakeVariables.txt
+BUILD_VERSION=171022
+# make sure this is a long random string
+# grab a "SHA 256-bit Key" from here: https://keygen.io/
+ACCOUNT_MANAGER_SECRET=
+# Should be the externally facing IP of your server host
+EXTERNAL_IP=localhost
+# Database values
+# Be careful with special characters here. It is more safe to use normal characters and/or numbers.
+MARIADB_USER=darkflame
+MARIADB_PASSWORD=SECRET_VALUE_CHANGE_ME
+MARIADB_ROOT_PASSWORD=SECRET_VALUE_CHANGE_ME
+MARIADB_DATABASE=darkflame

.gitattributes

@@ -0,0 +1 @@
+*.sh eol=lf

.gitignore

@@ -1,6 +1,7 @@
 temp/
 cmake-build-debug/
 RelWithDebInfo/
+docker/configs
 
 # Third party libraries
 thirdparty/mysql/
@@ -114,3 +115,8 @@ CMakeFiles/TargetDirectories.txt
 # clangd
 .cache
 thirdparty/zlib-1.2.11/
+
+.env
+docker/__pycache__
+
+docker-compose.override.yml

.gitmodules

@@ -10,3 +10,12 @@
 [submodule "thirdparty/libbcrypt"]
 	path = thirdparty/libbcrypt
 	url = https://github.com/trusch/libbcrypt.git
+[submodule "thirdparty/docker-utils"]
+	path = thirdparty/docker-utils
+	url = https://github.com/lcdr/utils.git
+[submodule "thirdparty/LUnpack"]
+	path = thirdparty/LUnpack
+	url = https://github.com/Xiphoseer/LUnpack.git
+[submodule "thirdparty/AccountManager"]
+	path = thirdparty/AccountManager
+	url = https://github.com/DarkflameUniverse/AccountManager

Docker.md

@@ -0,0 +1,38 @@
+# Run the Darkflame Server inside Docker
+
+## What you need
+
+- [Docker](https://docs.docker.com/get-docker/) (Docker Desktop or on Linux normal Docker)
+- [Docker Compose](https://docs.docker.com/compose/install/) (Included in Docker Desktop)
+- LEGO® Universe Client (packed or unpacked). Check the main [README](./README.md) for details on this.
+
+## Run server inside Docker
+
+1. Copy `.env.example` and save it as `.env` inside the root directory of this repository
+2. Edit the `.env` file and add your path to the root directory of your LEGO® Universe Client after `CLIENT_PATH=`
+3. Update other values in the `.env` file as needed (be sure to update passwords!)
+4. Run `docker compose up -d --build`
+5. Run `docker compose exec darkflame /app/MasterServer -a` and setup your admin account
+6. Follow the directions [here](https://github.com/DarkflameUniverse/AccountManager) to setup regular user accounts. The server will be accessible at: `http://<EXTERNAL_IP>:5000`
+7. Now you can see the output of the server with `docker compose logs -f --tail 100` or `docker compose logs -f --tail 100`. This can help you understand issues and there you can also see when the server finishes it's startup.
+8. You're ready to connect your client!
+
+**NOTE #1**: If you're running an older version of Docker, you may need to use the command `docker-compose` instead of `docker compose`.
+
+**NOTE #2**: To stop the server simply run `docker compose down` and to restart it just run `docker compose up -d` again. No need to run all the steps above every time.
+
+## Disable brickbuildfix
+
+If you don't need the http server running on port 80 do this:
+
+1. Create a file with the name `docker-compose.override.yml` in the root of the repository
+2. Paste this content:
+
+```yml
+services:
+  brickbuildfix:
+    profiles:
+      - donotstart
+```
+
+3. Now run `docker compose up -d`

Docker_Windows.md

@@ -0,0 +1,58 @@
+# Installation under Windows
+## First Run
+1. Navigate to the [Docker download page](https://www.docker.com/products/docker-desktop) and download docker.
+
+    ![Docker Download Page](docker/images/Docker_Download_Page.png)
+
+2. Once the file has finished downloading, run it and proceed through the installation. Make sure, "Install required Windows components for WSL 2" is checked.
+
+    ![Docker Desktop Installer Configuration](docker/images/Docker_Desktop_Installer_Configuration.png)
+
+3. If necessary, restart your computer.
+4. After the restart, Docker Desktop will automatically open. If it does not, simply start it like any other program.
+5. If a window "WSL 2 Installation is incomplete." pops up, follow the link and click "WSL2 Linux kernel update package for x64 machines". Run the downloaded file and once that finishes, click "Restart" in the Docker Desktop window.
+
+    ![WSL 2 download](docker/images/WSL_2_download.png)
+
+6. Wait until Docker Desktop has started. You may skip the tutorial.
+7. You may want to disable "Open Docker Dashboard at startup" in _Settings_ -> _General_
+
+    ![Disable Dashboard Autostart](docker/images/DD_General_Settings.png)
+
+8. Install [Git for Windows](https://git-scm.com/download/win). During the installation, simply confirming the defaults is sufficient.
+9. In the folder you wish to save the Server, right click and select "Git Bash Here".
+10. Type `git clone --recursive https://github.com/DarkflameUniverse/DarkflameServer`
+11. Once the command has completed (you can see you path again and can enter commands), close the window.
+12. Inside the downloaded folder, copy `.env.example` and name the copy `.env`
+13. Open `.env` with Notepad by right-clicking it and selecting _Open With_ -> _More apps_ -> _Notepad_.
+14. Change the text after `CLIENT_PATH=` to the location of your client. The folder you are pointing to must contain a folder called `client` which should contain the client files.
+    > If you need the extra performance, place the client files in `\\wsl$\<your linux OS>\...` to avoid working across file systems, see [Docker Best Practices](https://docs.docker.com/desktop/windows/wsl/#best-practices) and [WSL documentation](https://docs.microsoft.com/en-us/windows/wsl/filesystems#file-storage-and-performance-across-file-systems).
+
+15. Optionally, you can change the number after `BUILD_THREADS=` to the number of cores / threads your processor has. If your computer crashes while building, you can try to reduce this value.
+16. After `ACCOUNT_MANAGER_SECRET=` paste a "SHA 256-bit Key" from https://keygen.io/
+17. If you are not only hosting a local server, change the value after `EXTERNAL_IP=` to the external IP address of your computer.
+18. Change the two values `SECRET_VALUE_CHANGE_ME` to passwords only you know. Save and close the file.
+19. In the extracted folder hit Shift+Right Click and select "Open PowerShell window here".
+
+    ![Open PowerShell](docker/images/Open_Powershell.png)
+
+17. In the new window, paste (with right click) or type `docker compose up -d --build` and confirm with enter.
+18. Once you see the blinking cursor and the path again, setup has finished and the server is already running.
+
+    ![setup done](docker/images/setup_finished.png)
+
+19. Create an admin account by pasting `docker compose exec darkflame /app/MasterServer -a` and following the prompts.
+
+    ![admin account creation](docker/images/Account_Creation.png)
+
+20. You can now login with these credentials at `http://your_ip:5000` (replace your_ip with your external IP). There you can create your account for playing as well as generate keys for other people to join; use these at `http://your_ip:5000/activate`
+
+## Normal Use
+1. In Docker Desktop you should now see an entry `darkflameserver-main` and when you click on it all containers but `DarkflameSetup` should eventually be green. That means the server is running.
+
+    ![server running](docker/images/Docker_Compose_Finished.png)
+
+2. For troubleshooting, you can check the logs of the various parts by clicking their entry.
+3. You can start and stop the server with the corresponding buttons. Once all containers are grey, the server has shut down, and when all containers but `DarkflameSetup` are green, the server is running. Note that starting and stopping takes some time, please be patient.
+
+    ![start stop buttons](docker/images/DD_Server_Startstop.png)

dMasterServer/MasterServer.cpp

@@ -161,10 +161,16 @@ int main(int argc, char** argv) {
 	auto* masterLookupStatement = Database::CreatePreppedStmt("SELECT id FROM `servers` WHERE `name` = 'master'");
 	auto* result = masterLookupStatement->executeQuery();
 
+	auto master_server_ip = config.GetValue("master_ip");
+
+	if (master_server_ip == "") {
+		master_server_ip = Game::server->GetIP();
+	}
+
 	//If we found a server, update it's IP and port to the current one.
 	if (result->next()) {
 		auto* updateStatement = Database::CreatePreppedStmt("UPDATE `servers` SET `ip` = ?, `port` = ? WHERE `id` = ?");
-		updateStatement->setString(1, Game::server->GetIP());
+		updateStatement->setString(1, master_server_ip);
 		updateStatement->setInt(2, Game::server->GetPort());
 		updateStatement->setInt(3, result->getInt("id"));
 		updateStatement->execute();
@@ -173,7 +179,7 @@ int main(int argc, char** argv) {
 	else {
 		//If we didn't find a server, create one.
 		auto* insertStatement = Database::CreatePreppedStmt("INSERT INTO `servers` (`name`, `ip`, `port`, `state`, `version`) VALUES ('master', ?, ?, 0, 171023)");
-		insertStatement->setString(1, Game::server->GetIP());
+		insertStatement->setString(1, master_server_ip);
 		insertStatement->setInt(2, Game::server->GetPort());
 		insertStatement->execute();
 		delete insertStatement;

docker-compose.yml

@@ -0,0 +1,103 @@
+version: "3"
+
+services:
+  setup:
+    container_name: DarkflameSetup
+    build:
+      context: .
+      dockerfile: ./docker/setup.Dockerfile
+    environment:
+      - DATABASE=${MARIADB_DATABASE:-darkflame}
+      - DATABASE_HOST=database
+      - DATABASE_USER=${MARIADB_USER:-darkflame}
+      - DATABASE_PASSWORD=${MARIADB_PASSWORD:-darkflame}
+      - EXTERNAL_IP=${EXTERNAL_IP:-darkflame}
+    volumes:
+      - ${CLIENT_PATH:?missing_client_path}:/client
+      - shared_configs:/docker/
+
+  database:
+    container_name: DarkflameDatabase
+    build:
+      context: .
+      dockerfile: ./docker/database.Dockerfile
+    environment:
+      - MARIADB_USER=${MARIADB_USER:-darkflame}
+      - MARIADB_PASSWORD=${MARIADB_PASSWORD:-darkflame}
+      - MARIADB_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD:-darkflame}
+      - MARIADB_DATABASE=${MARIADB_DATABASE:-darkflame}
+    volumes:
+      - database:/var/lib/mysql
+    networks:
+      - darkflame
+    # You can expose these so that DB management tools can connect (WARNING: INSECURE)
+    # ports:
+    #   - 3306:3306
+
+  darkflame:
+    container_name: DarkflameServer
+    networks:
+      - darkflame
+    build:
+      context: .
+      dockerfile: ./docker/Dockerfile
+      args:
+        - BUILD_THREADS=${BUILD_THREADS:-1}
+        - BUILD_VERSION=${BUILD_VERSION:-171022}
+    volumes:
+      - ${CLIENT_PATH:?missing_client_path}:/client
+      - shared_configs:/shared_configs
+    depends_on:
+      - database
+    ports:
+      - "1001:1001/udp"
+      - "2000:2000/udp"
+      - "2005:2005/udp"
+      - "3000-3300:3000-3300/udp"
+
+  brickbuildfix:
+    container_name: DarkflameBrickBuildFix
+    networks:
+      - darkflame
+    build:
+      context: .
+      dockerfile: ./docker/brickfix.Dockerfile
+    ports:
+      - 80:80
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80"]
+      interval: 2m
+      timeout: 3s
+      retries: 3
+      start_period: 40s
+
+  account-manager:
+    container_name: DarkflameAccountManager
+    networks:
+      - darkflame
+    build:
+      context: .
+      dockerfile: ./docker/AccountManager.Dockerfile
+    environment:
+      - DATABASE=${MARIADB_DATABASE:-darkflame}
+      - DATABASE_HOST=database
+      - DATABASE_USER=${MARIADB_USER:-darkflame}
+      - DATABASE_PASSWORD=${MARIADB_PASSWORD:-darkflame}
+      - ACCOUNT_SECRET=${ACCOUNT_MANAGER_SECRET:?missing_account_secret}
+    ports:
+      - 5000:5000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000"]
+      interval: 2m
+      timeout: 3s
+      retries: 3
+      start_period: 40s
+    depends_on:
+      - database
+
+networks:
+  darkflame:
+
+volumes:
+  database:
+  shared_configs:

docker/AccountManager.Dockerfile

@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:1
+FROM python:3.10-alpine3.14
+
+WORKDIR /usr/local/share/AccountManager
+
+COPY ./thirdparty/AccountManager .
+
+ADD docker/credentials_example.py credentials.py
+ADD docker/resources_example.py resources.py
+
+RUN apk add libffi-dev build-base --no-cache && pip3 install -r requirements.txt
+
+EXPOSE 5000
+CMD python3 app.py

docker/Dockerfile

@@ -0,0 +1,59 @@
+FROM gcc:11 as build
+
+WORKDIR /build
+
+RUN --mount=type=cache,id=build-apt-cache,target=/var/cache/apt \
+    echo "Install build dependencies" && \
+    apt update && \
+    apt remove -y libmysqlcppconn7v5 libmysqlcppconn-dev && \
+    apt install cmake zlib1g zlib1g-dev unzip -yqq --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY dAuthServer/ /build/dAuthServer
+COPY dChatServer/ /build/dChatServer
+COPY dCommon/ /build/dCommon
+COPY dChatFilter/ /build/dChatFilter
+COPY dDatabase/ /build/dDatabase
+COPY dGame/ /build/dGame
+COPY dMasterServer/ /build/dMasterServer
+COPY dNet/ /build/dNet
+COPY dPhysics/ /build/dPhysics
+COPY dScripts/ /build/dScripts
+COPY dWorldServer/ /build/dWorldServer
+COPY dZoneManager/ /build/dZoneManager
+COPY migrations/ /build/migrations
+COPY resources/ /build/resources
+COPY thirdparty/ /build/thirdparty
+COPY vanity /build/vanity
+COPY tests/ /build/tests
+COPY .clang-* CMake* LICENSE /build/
+
+ARG BUILD_THREADS=1
+ARG BUILD_VERSION=171022
+
+RUN echo "Build server" && \
+    mkdir -p cmake_build && \
+    cd cmake_build && \
+    sed -i -e "s/171022/${BUILD_VERSION}/g" ../CMakeVariables.txt && \
+    cmake .. && \
+    make -j $BUILD_THREADS
+
+RUN unzip /build/resources/navmeshes.zip -d /build/cmake_build/res/maps
+
+FROM gcc:11 as runtime
+
+RUN --mount=type=cache,id=runtime-apt-cache,target=/var/cache/apt \
+    apt update && \
+    apt install sudo -yqq --no-install-recommends && \
+    apt remove -y libmysqlcppconn7v5 libmysqlcppconn-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=build /build/cmake_build /app
+
+RUN mkdir -p /build/cmake_build && ln -s /app/_deps /build/cmake_build/_deps
+
+COPY docker/start_server.sh /start_server.sh
+
+CMD [ "/start_server.sh" ]

docker/brickfix.Dockerfile

@@ -0,0 +1,5 @@
+# syntax=docker/dockerfile:1
+FROM python:3.9.9-slim
+WORKDIR /empty_dir
+EXPOSE 80
+CMD python -m http.server 80

docker/credentials_example.py

@@ -0,0 +1,4 @@
+import os
+
+DB_URL = f'mysql+pymysql://{os.environ["DATABASE_USER"]}:{os.environ["DATABASE_PASSWORD"]}@{os.environ["DATABASE_HOST"]}/{os.environ["DATABASE"]}'
+SECRET_KEY = os.environ["ACCOUNT_SECRET"]

docker/database.Dockerfile

@@ -0,0 +1,3 @@
+FROM mariadb:10.6
+
+COPY ./migrations/dlu /docker-entrypoint-initdb.d

docker/resources_example.py

@@ -0,0 +1,3 @@
+LOGO = 'logo/logo.png'
+PRIVACY_POLICY = 'policy/Privacy Policy.pdf'
+TERMS_OF_USE = 'policy/Terms of Use.pdf'

docker/setup.Dockerfile

@@ -0,0 +1,23 @@
+FROM rust:alpine3.14 as LUnpack
+
+WORKDIR /build_LUnpack
+
+COPY ./thirdparty/LUnpack .
+
+RUN apk add musl-dev --no-cache && cargo build --release
+
+FROM python:3.10-alpine3.14 as prep
+
+RUN apk add sqlite bash --no-cache
+
+WORKDIR /setup
+
+# copy needed files from repo
+COPY resources/ resources/
+COPY migrations/cdserver/ migrations/cdserver
+COPY --from=LUnpack /build_LUnpack/target/release/lunpack /usr/local/bin/lunpack
+ADD thirdparty/docker-utils/utils/*.py utils/
+
+COPY docker/setup.sh /setup.sh
+
+CMD [ "/setup.sh" ]

docker/setup.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# fail on first error
+set -e
+
+function update_ini() {
+    FILE="/docker/configs/$1"
+    KEY=$2
+    NEW_VALUE=$3
+    sed -i "/^$KEY=/s/=.*/=$NEW_VALUE/" $FILE
+}
+
+function update_database_ini_values_for() {
+    INI_FILE=$1
+
+    update_ini $INI_FILE mysql_host $DATABASE_HOST
+    update_ini $INI_FILE mysql_database $DATABASE
+    update_ini $INI_FILE mysql_username $DATABASE_USER
+    update_ini $INI_FILE mysql_password $DATABASE_PASSWORD
+    if [[ "$INI_FILE" != "worldconfig.ini" ]]; then
+        update_ini $INI_FILE external_ip $EXTERNAL_IP
+    fi
+}
+
+function update_ini_values() {
+    echo "Copying and updating config files"
+
+    mkdir -p /docker/configs
+    cp resources/masterconfig.ini /docker/configs/
+    cp resources/authconfig.ini /docker/configs/
+    cp resources/chatconfig.ini /docker/configs/
+    cp resources/worldconfig.ini /docker/configs/
+
+    update_ini worldconfig.ini chat_server_port $CHAT_SERVER_PORT
+    update_ini worldconfig.ini max_clients $MAX_CLIENTS
+
+    # always use the internal docker hostname
+    update_ini masterconfig.ini master_ip "darkflame"
+
+    update_database_ini_values_for masterconfig.ini
+    update_database_ini_values_for authconfig.ini
+    update_database_ini_values_for chatconfig.ini
+    update_database_ini_values_for worldconfig.ini
+}
+
+function fdb_to_sqlite() {
+    echo "Run fdb_to_sqlite"
+    python3 utils/fdb_to_sqlite.py /client/client/res/cdclient.fdb --sqlite_path /client/client/res/CDServer.sqlite
+
+    (
+        cd migrations/cdserver
+        readarray -d '' entries < <(printf '%s\0' *.sql | sort -zV)
+        for entry in "${entries[@]}"; do
+            echo "Execute $entry"
+            sqlite3 /client/client/res/CDServer.sqlite < $entry
+        done
+    )
+}
+
+update_ini_values
+
+if [[ ! -d "/client" ]]; then
+    echo "Client not found."
+    echo "Did you forget to mount the client into the \"/client\" directory?"
+    exit 1
+fi
+
+if [[ ! -f "/client/extracted" ]]; then
+    echo "Start client resource extraction"
+
+    touch globs.txt
+
+    echo "client/res/macros/**" >> globs.txt
+    echo "client/res/BrickModels/**" >> globs.txt
+    echo "client/res/maps/**" >> globs.txt
+    echo "*.fdb" >> globs.txt
+
+    lunpack -g ./globs.txt /client/
+
+    touch /client/extracted
+else
+    echo "Client already extracted. Skip this step..."
+    echo "If you want to force a re-extract, just delete the file called \"extracted\" in the client directory"
+fi
+
+if [[ ! -f "/client/migrated" ]]; then
+    echo "Start client db migration"
+
+    fdb_to_sqlite
+
+    touch /client/migrated
+else
+    echo "Client db already migrated. Skip this step..."
+    echo "If you want to force a re-migrate, just delete the file called \"migrated\" in the client directory"
+fi

docker/start_server.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+function symlink_client_files() {
+    echo "Creating symlinks for client files"
+    ln -s /client/client/res/macros/ /app/res/macros
+    ln -s /client/client/res/BrickModels/ /app/res/BrickModels
+    ln -s /client/client/res/chatplus_en_us.txt /app/res/chatplus_en_us.txt
+    ln -s /client/client/res/names/ /app/res/names
+    ln -s /client/client/res/CDServer.sqlite /app/res/CDServer.sqlite
+    ln -s /client/client/locale/locale.xml /app/locale/locale.xml
+    # need to iterate over entries in maps due to maps already being a directory with navmeshes/ in it
+    (
+        cd /client/client/res/maps
+        readarray -d '' entries < <(printf '%s\0' * | sort -zV)
+        for entry in "${entries[@]}"; do
+            ln -s /client/client/res/maps/$entry /app/res/maps/
+        done
+    )
+}
+
+function symlink_config_files() {
+    echo "Creating symlinks for config files"
+    rm /app/*.ini
+    ln -s /shared_configs/configs/authconfig.ini /app/authconfig.ini
+    ln -s /shared_configs/configs/chatconfig.ini /app/chatconfig.ini
+    ln -s /shared_configs/configs/masterconfig.ini /app/masterconfig.ini
+    ln -s /shared_configs/configs/worldconfig.ini /app/worldconfig.ini
+}
+
+# check to make sure the setup has completed
+while [ ! -f "/client/extracted" ] || [ ! -f "/client/migrated" ]; do
+    echo "Client setup not finished. Waiting for setup container to complete..."
+    sleep 5
+done
+
+if [[ ! -f "/app/initialized" ]]; then
+    # setup symlinks for volume files
+    symlink_client_files
+    symlink_config_files
+    # do not run symlinks more than once
+    touch /app/initialized
+else
+    echo "Server already initialized"
+fi
+
+# start the server
+echo "Starting MasterServer"
+./MasterServer
+tail -f /dev/null

resources/masterconfig.ini

@@ -7,6 +7,9 @@ mysql_password=
 # The public facing IP address. Can be 'localhost' for locally hosted servers
 external_ip=localhost
 
+# The internal ip of the master server
+master_ip=localhost
+
 # Port number
 port=2000
 

thirdparty/AccountManager

@@ -0,0 +1 @@
+Subproject commit 19e2a4fe9faba8ec2e40162e0019acd711967744

thirdparty/LUnpack

@@ -0,0 +1 @@
+Subproject commit f8d7e442a78910b298fe1cd5780f07c9c9285b8c

thirdparty/docker-utils

@@ -0,0 +1 @@
+Subproject commit 3f0129e0939ce5ccf41f0808dcbbe71a6243e37f