From 23c51af4375e276e691f06bb33a9a15b5798e85e Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 16 Sep 2022 23:14:53 -0400 Subject: [PATCH] Consolidated voxygen and server dynlibs into a common dynlib --- Cargo.lock | 36 ++-- Cargo.toml | 2 +- {voxygen => common}/dynlib/Cargo.toml | 2 +- {server => common}/dynlib/src/lib.rs | 0 server/agent/Cargo.toml | 4 +- server/agent/src/action_nodes.rs | 2 +- server/agent/src/lib.rs | 4 +- server/dynlib/Cargo.toml | 11 -- voxygen/anim/Cargo.toml | 4 +- voxygen/anim/src/lib.rs | 10 +- voxygen/dynlib/src/lib.rs | 272 -------------------------- voxygen/egui/Cargo.toml | 4 +- voxygen/egui/src/lib.rs | 10 +- 13 files changed, 34 insertions(+), 327 deletions(-) rename {voxygen => common}/dynlib/Cargo.toml (87%) rename {server => common}/dynlib/src/lib.rs (100%) delete mode 100644 server/dynlib/Cargo.toml delete mode 100644 voxygen/dynlib/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1fd6a104d2..7432504ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6600,6 +6600,16 @@ dependencies = [ "tracy-client", ] +[[package]] +name = "veloren-common-dynlib" +version = "0.1.0" +dependencies = [ + "find_folder", + "libloading 0.7.3", + "notify", + "tracing", +] + [[package]] name = "veloren-common-ecs" version = "0.10.0" @@ -6826,8 +6836,8 @@ dependencies = [ "vek 0.15.8", "veloren-common", "veloren-common-base", + "veloren-common-dynlib", "veloren-common-ecs", - "veloren-server-dynlib", ] [[package]] @@ -6861,16 +6871,6 @@ dependencies = [ "veloren-server", ] -[[package]] -name = "veloren-server-dynlib" -version = "0.1.0" -dependencies = [ - "find_folder", - "libloading 0.7.3", - "notify", - "tracing", -] - [[package]] name = "veloren-voxygen" version = "0.13.0" @@ -6961,7 +6961,7 @@ dependencies = [ "mimalloc", "vek 0.15.8", "veloren-common", - "veloren-voxygen-dynlib", + "veloren-common-dynlib", ] [[package]] @@ -6971,16 +6971,6 @@ dependencies = [ "veloren-voxygen-anim", ] -[[package]] -name = "veloren-voxygen-dynlib" -version = "0.1.0" -dependencies = [ - "find_folder", - "libloading 0.7.3", - "notify", - "tracing", -] - [[package]] name = "veloren-voxygen-egui" version = "0.9.0" @@ -6990,7 +6980,7 @@ dependencies = [ "lazy_static", "veloren-client", "veloren-common", - "veloren-voxygen-dynlib", + "veloren-common-dynlib", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eb182cfaaf..7eae22d0b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "common", "common/assets", "common/base", + "common/dynlib", "common/ecs", "common/net", "common/state", @@ -22,7 +23,6 @@ members = [ "voxygen/anim", "voxygen/anim/dyn", "voxygen/i18n", - "voxygen/dynlib", "voxygen/egui", "voxygen/egui/dyn", "world", diff --git a/voxygen/dynlib/Cargo.toml b/common/dynlib/Cargo.toml similarity index 87% rename from voxygen/dynlib/Cargo.toml rename to common/dynlib/Cargo.toml index 5eccc940d9..ba7cd7ffde 100644 --- a/voxygen/dynlib/Cargo.toml +++ b/common/dynlib/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "veloren-voxygen-dynlib" +name = "veloren-common-dynlib" version = "0.1.0" authors = ["Ben Wallis "] edition = "2021" diff --git a/server/dynlib/src/lib.rs b/common/dynlib/src/lib.rs similarity index 100% rename from server/dynlib/src/lib.rs rename to common/dynlib/src/lib.rs diff --git a/server/agent/Cargo.toml b/server/agent/Cargo.toml index c94adf6daf..6363701404 100644 --- a/server/agent/Cargo.toml +++ b/server/agent/Cargo.toml @@ -5,14 +5,14 @@ edition = "2021" version = "0.1.0" [features] -use-dyn-lib = ["server-dynlib"] +use-dyn-lib = ["common-dynlib"] be-dyn-lib = [] [dependencies] common = {package = "veloren-common", path = "../../common"} common-base = { package = "veloren-common-base", path = "../../common/base" } common-ecs = { package = "veloren-common-ecs", path = "../../common/ecs" } -server-dynlib = {package = "veloren-server-dynlib", path = "../dynlib", optional = true} +common-dynlib = {package = "veloren-common-dynlib", path = "../../common/dynlib", optional = true} specs = { version = "0.18", features = ["shred-derive"] } vek = { version = "0.15.8", features = ["serde"] } diff --git a/server/agent/src/action_nodes.rs b/server/agent/src/action_nodes.rs index 05fc1882d2..cd0272d15a 100644 --- a/server/agent/src/action_nodes.rs +++ b/server/agent/src/action_nodes.rs @@ -717,7 +717,7 @@ impl<'a> AgentData<'a> { let lib = &lock.as_ref().unwrap().lib; const ATTACK_FN: &[u8] = b"attack_inner\0"; - let attack_fn: server_dynlib::Symbol< + let attack_fn: common_dynlib::Symbol< fn(&Self, &mut Agent, &mut Controller, &TargetData, &ReadData), > = unsafe { lib.get(ATTACK_FN) }.unwrap_or_else(|e| { panic!( diff --git a/server/agent/src/lib.rs b/server/agent/src/lib.rs index 70baa16340..d30254cb22 100644 --- a/server/agent/src/lib.rs +++ b/server/agent/src/lib.rs @@ -12,12 +12,12 @@ pub mod data; pub mod util; #[cfg(feature = "use-dyn-lib")] -use {lazy_static::lazy_static, server_dynlib::LoadedLib, std::sync::Arc, std::sync::Mutex}; +use {common_dynlib::LoadedLib, lazy_static::lazy_static, std::sync::Arc, std::sync::Mutex}; #[cfg(feature = "use-dyn-lib")] lazy_static! { pub static ref LIB: Arc>> = - server_dynlib::init("veloren-server-agent", "veloren-server-agent-dyn", "agent"); + common_dynlib::init("veloren-server-agent", "veloren-server-agent-dyn", "agent"); } #[cfg(feature = "use-dyn-lib")] diff --git a/server/dynlib/Cargo.toml b/server/dynlib/Cargo.toml deleted file mode 100644 index 01a3a381c8..0000000000 --- a/server/dynlib/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "veloren-server-dynlib" -version = "0.1.0" -authors = ["Samuel Keiffer "] -edition = "2021" - -[dependencies] -find_folder = {version = "0.3.0"} -libloading = {version = "0.7"} -notify = {version = "5.0.0"} -tracing = "0.1" diff --git a/voxygen/anim/Cargo.toml b/voxygen/anim/Cargo.toml index 9c4a7a2b07..2f3afd8bd4 100644 --- a/voxygen/anim/Cargo.toml +++ b/voxygen/anim/Cargo.toml @@ -5,7 +5,7 @@ name = "veloren-voxygen-anim" version = "0.10.0" [features] -use-dyn-lib = ["lazy_static", "voxygen-dynlib"] +use-dyn-lib = ["lazy_static", "common-dynlib"] be-dyn-lib = [] simd = ["vek/platform_intrinsics"] @@ -16,7 +16,7 @@ common = {package = "veloren-common", path = "../../common"} # inline_tweak = "1.0.2" bytemuck = { version = "1.4", features=["derive"] } vek = {version = "0.15.8", features = ["serde"]} -voxygen-dynlib = {package = "veloren-voxygen-dynlib", path = "../dynlib", optional = true} +common-dynlib = {package = "veloren-common-dynlib", path = "../../common/dynlib", optional = true} # Hot Reloading lazy_static = {version = "1.4.0", optional = true} diff --git a/voxygen/anim/src/lib.rs b/voxygen/anim/src/lib.rs index e3aa8d44f9..317c8c8c12 100644 --- a/voxygen/anim/src/lib.rs +++ b/voxygen/anim/src/lib.rs @@ -77,8 +77,8 @@ use bytemuck::{Pod, Zeroable}; use common::comp::tool::ToolKind; #[cfg(feature = "use-dyn-lib")] use { - lazy_static::lazy_static, std::ffi::CStr, std::sync::Arc, std::sync::Mutex, - voxygen_dynlib::LoadedLib, + common_dynlib::LoadedLib, lazy_static::lazy_static, std::ffi::CStr, std::sync::Arc, + std::sync::Mutex, }; type MatRaw = [[f32; 4]; 4]; @@ -99,7 +99,7 @@ pub type Bone = Transform; #[cfg(feature = "use-dyn-lib")] lazy_static! { static ref LIB: Arc>> = - voxygen_dynlib::init("veloren-voxygen-anim", "veloren-voxygen-anim-dyn", "anim"); + common_dynlib::init("veloren-voxygen-anim", "veloren-voxygen-anim-dyn", "anim"); } #[cfg(feature = "use-dyn-lib")] @@ -179,7 +179,7 @@ pub trait Skeleton: Send + Sync + 'static { let lock = LIB.lock().unwrap(); let lib = &lock.as_ref().unwrap().lib; - let compute_fn: voxygen_dynlib::Symbol< + let compute_fn: common_dynlib::Symbol< fn(&Self, Mat4, &mut [FigureBoneData; MAX_BONE_COUNT], Self::Body) -> Offsets, > = unsafe { lib.get(Self::COMPUTE_FN) }.unwrap_or_else(|e| { panic!( @@ -247,7 +247,7 @@ pub trait Animation { let lock = LIB.lock().unwrap(); let lib = &lock.as_ref().unwrap().lib; - let update_fn: voxygen_dynlib::Symbol< + let update_fn: common_dynlib::Symbol< fn( &Self::Skeleton, Self::Dependency<'a>, diff --git a/voxygen/dynlib/src/lib.rs b/voxygen/dynlib/src/lib.rs deleted file mode 100644 index d763e1ab03..0000000000 --- a/voxygen/dynlib/src/lib.rs +++ /dev/null @@ -1,272 +0,0 @@ -use libloading::Library; -use notify::{recommended_watcher, EventKind, RecursiveMode, Watcher}; -use std::{ - process::{Command, Stdio}, - sync::{mpsc, Mutex}, - time::Duration, -}; - -use find_folder::Search; -use std::{ - env, - env::consts::{DLL_PREFIX, DLL_SUFFIX}, - path::{Path, PathBuf}, - sync::Arc, -}; -use tracing::{debug, error, info}; - -// Re-exports -pub use libloading::Symbol; - -/// LoadedLib holds a loaded dynamic library and the location of library file -/// with the appropriate OS specific name and extension i.e. -/// `libvoxygen_anim_dyn_active.dylib`, `voxygen_anim_dyn_active.dll`. -/// -/// # NOTE -/// DOES NOT WORK ON MACOS, due to some limitations with hot-reloading the -/// `.dylib`. -pub struct LoadedLib { - /// Loaded library. - pub lib: Library, - /// Path to the library. - pub lib_path: PathBuf, -} - -impl LoadedLib { - /// Compile and load the dynamic library - /// - /// This is necessary because the very first time you use hot reloading you - /// wont have the library, so you can't load it until you have compiled it! - fn compile_load(dyn_package: &str) -> Self { - #[cfg(target_os = "macos")] - error!("The hot reloading feature does not work on macos."); - - // Compile - if !compile(dyn_package) { - panic!("{} compile failed.", dyn_package); - } else { - info!("{} compile succeeded.", dyn_package); - } - - copy(&LoadedLib::determine_path(dyn_package), dyn_package); - - Self::load(dyn_package) - } - - /// Load a library from disk. - /// - /// Currently this is pretty fragile, it gets the path of where it thinks - /// the dynamic library should be and tries to load it. It will panic if it - /// is missing. - fn load(dyn_package: &str) -> Self { - let lib_path = LoadedLib::determine_path(dyn_package); - - // Try to load the library. - let lib = match unsafe { Library::new(lib_path.clone()) } { - Ok(lib) => lib, - Err(e) => panic!( - "Tried to load dynamic library from {:?}, but it could not be found. A potential \ - reason is we may require a special case for your OS so we can find it. {:?}", - lib_path, e - ), - }; - - Self { lib, lib_path } - } - - /// Determine the path to the dynamic library based on the path of the - /// current executable. - fn determine_path(dyn_package: &str) -> PathBuf { - let current_exe = env::current_exe(); - - // If we got the current_exe, we need to go up a level and then down - // in to debug (in case we were in release or another build dir). - let mut lib_path = match current_exe { - Ok(mut path) => { - // Remove the filename to get the directory. - path.pop(); - - // Search for the debug directory. - let dir = Search::ParentsThenKids(1, 1) - .of(path) - .for_folder("debug") - .expect( - "Could not find the debug build directory relative to the current \ - executable.", - ); - - debug!(?dir, "Found the debug build directory."); - dir - }, - Err(e) => { - panic!( - "Could not determine the path of the current executable, this is needed to \ - hot-reload the dynamic library. {:?}", - e - ); - }, - }; - - // Determine the platform specific path and push it onto our already - // established target/debug dir. - lib_path.push(active_file(dyn_package)); - - lib_path - } -} - -/// Initialise a watcher. -/// -/// This will search for the directory named `package_source_dir` and watch the -/// files within it for any changes. -pub fn init( - package: &'static str, - dyn_package: &'static str, - package_source_dir: &'static str, -) -> Arc>> { - let lib_storage = Arc::new(Mutex::new(Some(LoadedLib::compile_load(dyn_package)))); - - // TODO: use crossbeam - let (reload_send, reload_recv) = mpsc::channel(); - - // Start watcher - let mut watcher = recommended_watcher(move |res| event_fn(res, &reload_send)).unwrap(); - - // Search for the source directory of the package being hot-reloaded. - let watch_dir = Search::Kids(1) - .for_folder(package_source_dir) - .unwrap_or_else(|_| { - panic!( - "Could not find the {} crate directory relative to the current directory", - package_source_dir - ) - }); - - watcher.watch(&watch_dir, RecursiveMode::Recursive).unwrap(); - - // Start reloader that watcher signals - // "Debounces" events since I can't find the option to do this in the latest - // `notify` - let lib_storage_clone = Arc::clone(&lib_storage); - std::thread::Builder::new() - .name(format!("{}_hotreload_watcher", package)) - .spawn(move || { - let mut modified_paths = std::collections::HashSet::new(); - while let Ok(path) = reload_recv.recv() { - modified_paths.insert(path); - // Wait for any additional modify events before reloading - while let Ok(path) = reload_recv.recv_timeout(Duration::from_millis(300)) { - modified_paths.insert(path); - } - - info!( - ?modified_paths, - "Hot reloading {} because files in `{}` modified.", package, package_source_dir - ); - - hotreload(dyn_package, &lib_storage_clone); - } - }) - .unwrap(); - - // Let the watcher live forever - std::mem::forget(watcher); - - lib_storage -} - -fn compiled_file(dyn_package: &str) -> String { dyn_lib_file(dyn_package, false) } - -fn active_file(dyn_package: &str) -> String { dyn_lib_file(dyn_package, true) } - -fn dyn_lib_file(dyn_package: &str, active: bool) -> String { - format!( - "{}{}{}{}", - DLL_PREFIX, - dyn_package.replace('-', "_"), - if active { "_active" } else { "" }, - DLL_SUFFIX - ) -} - -/// Event function to hotreload the dynamic library -/// -/// This is called by the watcher to filter for modify events on `.rs` files -/// before sending them back. -fn event_fn(res: notify::Result, sender: &mpsc::Sender) { - match res { - Ok(event) => { - if let EventKind::Modify(_) = event.kind { - event - .paths - .iter() - .filter(|p| p.extension().map(|e| e == "rs").unwrap_or(false)) - .map(|p| p.to_string_lossy().into_owned()) - // Signal reloader - .for_each(|p| { let _ = sender.send(p); }); - } - }, - Err(e) => error!(?e, "hotreload watcher error."), - } -} - -/// Hotreload the dynamic library -/// -/// This will reload the dynamic library by first internally calling compile -/// and then reloading the library. -fn hotreload(dyn_package: &str, loaded_lib: &Mutex>) { - // Do nothing if recompile failed. - if compile(dyn_package) { - let mut lock = loaded_lib.lock().unwrap(); - - // Close lib. - let loaded_lib = lock.take().unwrap(); - loaded_lib.lib.close().unwrap(); - copy(&loaded_lib.lib_path, dyn_package); - - // Open new lib. - *lock = Some(LoadedLib::load(dyn_package)); - - info!("Updated {}.", dyn_package); - } -} - -/// Recompile the dyn package -/// -/// Returns `false` if the compile failed. -fn compile(dyn_package: &str) -> bool { - let output = Command::new("cargo") - .stderr(Stdio::inherit()) - .stdout(Stdio::inherit()) - .arg("build") - .arg("--package") - .arg(dyn_package) - .arg("--features") - .arg(format!("{}/be-dyn-lib", dyn_package)) - .output() - .unwrap(); - - output.status.success() -} - -/// Copy the lib file, so we have an `_active` copy. -/// -/// We do this for all OS's although it is only strictly necessary for windows. -/// The reason we do this is to make the code easier to understand and debug. -fn copy(lib_path: &Path, dyn_package: &str) { - // Use the platform specific names. - let lib_compiled_path = lib_path.with_file_name(compiled_file(dyn_package)); - let lib_output_path = lib_path.with_file_name(active_file(dyn_package)); - - // Get the path to where the lib was compiled to. - debug!(?lib_compiled_path, ?lib_output_path, "Moving."); - - // Copy the library file from where it is output, to where we are going to - // load it from i.e. lib_path. - std::fs::copy(&lib_compiled_path, &lib_output_path).unwrap_or_else(|err| { - panic!( - "Failed to rename dynamic library from {:?} to {:?}. {:?}", - lib_compiled_path, lib_output_path, err - ) - }); -} diff --git a/voxygen/egui/Cargo.toml b/voxygen/egui/Cargo.toml index 1a32adbf90..cc1c9b0a5f 100644 --- a/voxygen/egui/Cargo.toml +++ b/voxygen/egui/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" version = "0.9.0" [features] -use-dyn-lib = ["voxygen-dynlib"] +use-dyn-lib = ["common-dynlib"] be-dyn-lib = [] [dependencies] @@ -14,6 +14,6 @@ common = {package = "veloren-common", path = "../../common"} egui = "0.12" egui_winit_platform = "0.8" lazy_static = "1.4.0" -voxygen-dynlib = {package = "veloren-voxygen-dynlib", path = "../dynlib", optional = true} +common-dynlib = {package = "veloren-common-dynlib", path = "../../common/dynlib", optional = true} diff --git a/voxygen/egui/src/lib.rs b/voxygen/egui/src/lib.rs index 251daf4244..c096592bb3 100644 --- a/voxygen/egui/src/lib.rs +++ b/voxygen/egui/src/lib.rs @@ -30,14 +30,14 @@ use egui_winit_platform::Platform; use std::time::Duration; #[cfg(feature = "use-dyn-lib")] use { - lazy_static::lazy_static, std::ffi::CStr, std::sync::Arc, std::sync::Mutex, - voxygen_dynlib::LoadedLib, + common_dynlib::LoadedLib, lazy_static::lazy_static, std::ffi::CStr, std::sync::Arc, + std::sync::Mutex, }; #[cfg(feature = "use-dyn-lib")] lazy_static! { static ref LIB: Arc>> = - voxygen_dynlib::init("veloren-voxygen-egui", "veloren-voxygen-egui-dyn", "egui"); + common_dynlib::init("veloren-voxygen-egui", "veloren-voxygen-egui-dyn", "egui"); } #[cfg(feature = "use-dyn-lib")] @@ -170,7 +170,7 @@ pub fn maintain( let lock = LIB.lock().unwrap(); let lib = &lock.as_ref().unwrap().lib; - let maintain_fn: voxygen_dynlib::Symbol< + let maintain_fn: common_dynlib::Symbol< fn( &mut Platform, &mut EguiInnerState, @@ -614,7 +614,7 @@ fn selected_entity_window( .spacing([40.0, 4.0]) .max_col_width(100.0) .striped(true) - // Apparently, if the #[rustfmt::skip] is in front of the closure scope, rust-analyzer can't + // Apparently, if the #[rustfmt::skip] is in front of the closure scope, rust-analyzer can't // parse the code properly. Things will *sometimes* work if the skip is on the other side of // the opening bracket (even though that should only skip formatting the first line of the // closure), but things as arbitrary as adding a comment to the code cause it to be formatted