From 12ee21a2891521d01f74c6543755353c563bb18f Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sat, 14 Oct 2023 22:35:06 +0200 Subject: [PATCH 01/11] first part of load items from plugin --- Cargo.lock | 2 + common/assets/Cargo.toml | 6 +- common/assets/src/lib.rs | 64 +++++++- common/assets/src/plugin_cache.rs | 148 +++++++++++++++++ common/assets/src/tar_source.rs | 262 ++++++++++++++++++++++++++++++ common/state/src/plugin/mod.rs | 10 +- voxygen/Cargo.toml | 2 +- voxygen/src/scene/figure/load.rs | 72 +++++++- 8 files changed, 557 insertions(+), 9 deletions(-) create mode 100644 common/assets/src/plugin_cache.rs create mode 100644 common/assets/src/tar_source.rs diff --git a/Cargo.lock b/Cargo.lock index 839fc422db..15686fcdd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6872,10 +6872,12 @@ version = "0.10.0" dependencies = [ "assets_manager", "dot_vox", + "hashbrown 0.13.2", "image", "lazy_static", "ron 0.8.1", "serde", + "tar", "tracing", "walkdir", "wavefront", diff --git a/common/assets/Cargo.toml b/common/assets/Cargo.toml index 2e3965c462..1730ad25e4 100644 --- a/common/assets/Cargo.toml +++ b/common/assets/Cargo.toml @@ -13,13 +13,17 @@ dot_vox = "5.1" wavefront = "0.2" # TODO: Use vertex-colors branch when we have models that have them image = { workspace = true } tracing = { workspace = true } +tar = { version = "0.4.37", optional = true } # asset tweak serde = { workspace = true, optional = true } +hashbrown = { workspace = true, optional = true } [dev-dependencies] walkdir = "2.3.2" [features] hot-reloading = ["assets_manager/hot-reloading"] -asset_tweak = ["serde", "hot-reloading"] +asset_tweak = ["dep:serde", "hot-reloading"] +hashbrown = ["dep:hashbrown"] +plugins = ["dep:serde", "dep:tar"] diff --git a/common/assets/src/lib.rs b/common/assets/src/lib.rs index 9b8f6102be..abc3e659ef 100644 --- a/common/assets/src/lib.rs +++ b/common/assets/src/lib.rs @@ -4,7 +4,13 @@ use dot_vox::DotVoxData; use image::DynamicImage; use lazy_static::lazy_static; -use std::{borrow::Cow, path::PathBuf, sync::Arc}; +use std::{ + borrow::Cow, + collections::HashMap, + hash::{BuildHasher, Hash}, + path::PathBuf, + sync::Arc, +}; pub use assets_manager::{ asset::{DirLoadable, Ron}, @@ -16,13 +22,15 @@ pub use assets_manager::{ }; mod fs; +mod plugin_cache; +#[cfg(feature = "plugins")] mod tar_source; mod walk; +pub use plugin_cache::register_tar; pub use walk::{walk_tree, Walk}; lazy_static! { /// The HashMap where all loaded assets are stored in. - static ref ASSETS: AssetCache = - AssetCache::with_source(fs::FileSystem::new().unwrap()); + static ref ASSETS: plugin_cache::CombinedCache = plugin_cache::CombinedCache::new().unwrap(); } #[cfg(feature = "hot-reloading")] @@ -209,6 +217,56 @@ impl Loader for ObjAssetLoader { } } +pub trait Concatenate { + fn concatenate(self, b: Self) -> Self; +} + +impl Concatenate for HashMap { + fn concatenate(mut self, b: Self) -> Self { + self.extend(b); + self + } +} + +impl Concatenate for Vec { + fn concatenate(mut self, b: Self) -> Self { + self.extend(b); + self + } +} + +#[cfg(feature = "hashbrown")] +impl Concatenate for hashbrown::HashMap { + fn concatenate(mut self, b: Self) -> Self { + self.extend(b); + self + } +} + +impl Concatenate for Ron { + fn concatenate(self, _b: Self) -> Self { todo!() } +} + +/// This wrapper combines several RON files from multiple sources +pub struct MultiRon(pub T); + +impl Clone for MultiRon { + fn clone(&self) -> Self { Self(self.0.clone()) } + + fn clone_from(&mut self, source: &Self) { self.0.clone_from(&source.0) } +} + +impl Compound for MultiRon +where + T: for<'de> serde::Deserialize<'de> + Send + Sync + 'static + Concatenate, +{ + fn load(_cache: AnyCache, id: &SharedString) -> Result { + ASSETS + .combine(|cache: AnyCache| as Compound>::load(cache, id).map(|r| r.0)) + .map(MultiRon) + } +} + /// Return path to repository root by searching 10 directories back pub fn find_root() -> Option { std::env::current_dir().map_or(None, |path| { diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs new file mode 100644 index 0000000000..e626dbdba2 --- /dev/null +++ b/common/assets/src/plugin_cache.rs @@ -0,0 +1,148 @@ +use std::{path::PathBuf, sync::RwLock}; + +use crate::Concatenate; + +use super::{fs::FileSystem, tar_source::Tar}; +use assets_manager::{ + source::{FileContent, Source}, + AnyCache, AssetCache, BoxedError, +}; +use lazy_static::lazy_static; + +struct PluginEntry { + path: PathBuf, + cache: AssetCache, +} + +lazy_static! { + static ref PLUGIN_LIST: RwLock> = RwLock::new(Vec::new()); +} + +pub fn register_tar(path: PathBuf) -> Result<(), Box> { + let tar_source = Tar::from_path(&path)?; + println!("Tar {:?} {:?}", path, tar_source); + let cache = AssetCache::with_source(tar_source); + PLUGIN_LIST.write()?.push(PluginEntry { path, cache }); + Ok(()) +} + +/// The source combining filesystem and plugins (typically used via +/// CombinedCache) +#[derive(Debug, Clone)] +pub struct CombinedSource { + fs: FileSystem, +} + +impl CombinedSource { + pub fn new() -> std::io::Result { + Ok(Self { + fs: FileSystem::new()?, + }) + } +} + +impl CombinedSource { + fn read_multiple(&self, id: &str, ext: &str) -> Vec<(Option, FileContent<'_>)> { + let mut result = Vec::new(); + if let Ok(file_entry) = self.fs.read(id, ext) { + result.push((None, file_entry)); + } + if let Ok(guard) = PLUGIN_LIST.read() { + for (n, p) in guard.iter().enumerate() { + if let Ok(entry) = p.cache.raw_source().read(id, ext) { + result.push((Some(n), match entry { + FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), + FileContent::Buffer(b) => FileContent::Buffer(b), + FileContent::Owned(s) => { + FileContent::Buffer(Vec::from(s.as_ref().as_ref())) + }, + })); + } + } + } + result + } + + // we don't want to keep the lock, so we clone + fn plugin_path(&self, index: Option) -> Option { + if let Some(index) = index { + PLUGIN_LIST + .read() + .ok() + .and_then(|p| p.get(index).map(|p| p.path.clone())) + } else { + None + } + } +} + +impl Source for CombinedSource { + fn read(&self, id: &str, ext: &str) -> std::io::Result> { + // we could shortcut on fs if we dont want to check for conflicts + let mut entries = self.read_multiple(id, ext); + if entries.is_empty() { + Err(std::io::ErrorKind::NotFound.into()) + } else { + if entries.len() > 1 { + tracing::error!( + "Duplicate asset {id} in read, plugins {:?} + {:?}", + self.plugin_path(entries[0].0), + self.plugin_path(entries[1].0) + ); + } + Ok(entries.swap_remove(0).1) + } + } + + fn read_dir( + &self, + id: &str, + f: &mut dyn FnMut(assets_manager::source::DirEntry), + ) -> std::io::Result<()> { + // TODO: we should combine the sources + self.fs.read_dir(id, f) + } + + fn exists(&self, entry: assets_manager::source::DirEntry) -> bool { + self.fs.exists(entry) + || PLUGIN_LIST + .read() + .map(|p| p.iter().any(|p| p.cache.raw_source().exists(entry))) + .unwrap_or_default() + } + + fn make_source(&self) -> Option> { None } +} + +/// A cache combining filesystem and plugin assets +pub struct CombinedCache(AssetCache); + +impl CombinedCache { + pub fn new() -> std::io::Result { + CombinedSource::new().map(|s| Self(AssetCache::with_source(s))) + } + + /// combine objects from filesystem and plugins + pub fn combine( + &self, + load_from: impl Fn(AnyCache) -> Result, + ) -> Result { + let mut result = load_from(self.as_any_cache()); + for i in PLUGIN_LIST.read().unwrap().iter() { + if let Ok(b) = load_from(i.cache.as_any_cache()) { + result = if let Ok(a) = result { + Ok(a.concatenate(b)) + } else { + Ok(b) + }; + } + } + result + } +} + +impl std::ops::Deref for CombinedCache { + type Target = AssetCache; + + fn deref(&self) -> &Self::Target { &self.0 } +} diff --git a/common/assets/src/tar_source.rs b/common/assets/src/tar_source.rs new file mode 100644 index 0000000000..259d184f42 --- /dev/null +++ b/common/assets/src/tar_source.rs @@ -0,0 +1,262 @@ +use assets_manager::source::{DirEntry, FileContent, Source}; +use hashbrown::HashMap; +use tar::EntryType; + +use std::{ + fmt, + fs::File, + hash, io, + os::unix::prelude::FileExt, + path::{self, Path, PathBuf}, +}; + +// derived from the zip source from assets_manager + +#[inline] +pub(crate) fn extension_of(path: &Path) -> Option<&str> { + match path.extension() { + Some(ext) => ext.to_str(), + None => Some(""), + } +} + +#[derive(Clone, Hash, PartialEq, Eq)] +struct FileDesc(String, String); + +/// This hack enables us to use a `(&str, &str)` as a key for an HashMap without +/// allocating a `FileDesc` +trait FileKey { + fn id(&self) -> &str; + fn ext(&self) -> &str; +} + +impl FileKey for FileDesc { + fn id(&self) -> &str { &self.0 } + + fn ext(&self) -> &str { &self.1 } +} + +impl FileKey for (&'_ str, &'_ str) { + fn id(&self) -> &str { self.0 } + + fn ext(&self) -> &str { self.1 } +} + +impl<'a> std::borrow::Borrow for FileDesc { + fn borrow(&self) -> &(dyn FileKey + 'a) { self } +} + +impl PartialEq for dyn FileKey + '_ { + fn eq(&self, other: &Self) -> bool { self.id() == other.id() && self.ext() == other.ext() } +} + +impl Eq for dyn FileKey + '_ {} + +impl hash::Hash for dyn FileKey + '_ { + fn hash(&self, hasher: &mut H) { + self.id().hash(hasher); + self.ext().hash(hasher); + } +} + +impl fmt::Debug for FileDesc { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("FileDesc") + .field("id", &self.0) + .field("ext", &self.1) + .finish() + } +} + +/// An entry in a archive directory. +#[derive(Debug)] +enum OwnedEntry { + File(FileDesc), + // Dir(String), +} + +impl OwnedEntry { + fn as_dir_entry(&self) -> DirEntry { + match self { + OwnedEntry::File(FileDesc(desc0, desc1)) => DirEntry::File(desc0, desc1), + // OwnedEntry::Dir(id) => DirEntry::Directory(id), + } + } +} + +/// Build ids from components. +/// +/// Using this allows to easily reuse buffers when building several ids in a +/// row, and thus to avoid repeated allocations. +#[derive(Default)] +struct IdBuilder { + segments: Vec, + len: usize, +} + +impl IdBuilder { + /// Pushs a segment in the builder. + #[inline] + fn push(&mut self, s: &str) { + match self.segments.get_mut(self.len) { + Some(seg) => { + seg.clear(); + seg.push_str(s); + }, + None => self.segments.push(s.to_owned()), + } + self.len += 1; + } + + /// Joins segments to build a id. + #[inline] + fn join(&self) -> String { self.segments[..self.len].join(".") } + + /// Resets the builder without freeing buffers. + #[inline] + fn reset(&mut self) { self.len = 0; } +} + +/// Register a file of an archive in maps. +fn register_file( + path: &Path, + position: usize, + length: usize, + files: &mut HashMap, + dirs: &mut HashMap>, + id_builder: &mut IdBuilder, +) { + id_builder.reset(); + + // Parse the path and register it. + // The closure is used as a cheap `try` block. + let ok = (|| { + // Fill `id_builder` from the parent's components + let parent = path.parent()?; + for comp in parent.components() { + match comp { + path::Component::Normal(s) => { + let segment = s.to_str()?; + if segment.contains('.') { + return None; + } + id_builder.push(segment); + }, + _ => return None, + } + } + + // Build the ids of the file and its parent. + let parent_id = id_builder.join(); + id_builder.push(path.file_stem()?.to_str()?); + let id = id_builder.join(); + + // Register the file in the maps. + let ext = extension_of(path)?.to_owned(); + let desc = FileDesc(id, ext); + files.insert(desc.clone(), (position, length)); + let entry = OwnedEntry::File(desc); + dirs.entry(parent_id).or_default().push(entry); + + Some(()) + })() + .is_some(); + + if !ok { + tracing::warn!("Unsupported path in tar archive: {path:?}"); + } +} + +enum Backend { + File(PathBuf), + // Buffer(&'static [u8]), +} + +impl Backend { + fn read(&self, pos: usize, len: usize) -> std::io::Result> { + match self { + Backend::File(path) => File::open(path).and_then(|file| { + let mut result = vec![0; len]; + file.read_at(result.as_mut_slice(), pos as u64) + .map(move |_bytes| result) + }), + // Backend::Buffer(_) => todo!(), + } + } +} + +pub struct Tar { + files: HashMap, + dirs: HashMap>, + backend: Backend, +} + +impl Tar { + /// Creates a `Tar` from a file + pub fn from_path(path: &Path) -> io::Result { + let file = File::open(path)?; + let mut tar = tar::Archive::new(file); + let contents = tar + .entries() + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mut files = HashMap::with_capacity(contents.size_hint().0); + let mut dirs = HashMap::new(); + let mut id_builder = IdBuilder::default(); + for e in contents.flatten() { + if matches!(e.header().entry_type(), EntryType::Regular) { + register_file( + e.path().map_err(io::Error::other)?.as_ref(), + e.raw_file_position() as usize, + e.size() as usize, + &mut files, + &mut dirs, + &mut id_builder, + ); + } + } + Ok(Tar { + files, + dirs, + backend: Backend::File(path.to_path_buf()), + }) + } +} + +impl Source for Tar { + fn read(&self, id: &str, ext: &str) -> io::Result { + let key: &dyn FileKey = &(id, ext); + let id = *self + .files + .get(key) + .or_else(|| { + // also accept assets within the assets dir for now + let with_prefix = "assets.".to_string() + id; + let prefixed_key: &dyn FileKey = &(with_prefix.as_str(), ext); + self.files.get(prefixed_key) + }) + .ok_or(io::ErrorKind::NotFound)?; + self.backend.read(id.0, id.1).map(FileContent::Buffer) + } + + fn read_dir(&self, id: &str, f: &mut dyn FnMut(DirEntry)) -> io::Result<()> { + let dir = self.dirs.get(id).ok_or(io::ErrorKind::NotFound)?; + dir.iter().map(OwnedEntry::as_dir_entry).for_each(f); + Ok(()) + } + + fn exists(&self, entry: DirEntry) -> bool { + match entry { + DirEntry::File(id, ext) => self.files.contains_key(&(id, ext) as &dyn FileKey), + DirEntry::Directory(id) => self.dirs.contains_key(id), + } + } +} + +impl fmt::Debug for Tar { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Tar") + .field("files", &self.files) + .field("dirs", &self.dirs) + .finish() + } +} diff --git a/common/state/src/plugin/mod.rs b/common/state/src/plugin/mod.rs index 5bbd434e89..0db8359221 100644 --- a/common/state/src/plugin/mod.rs +++ b/common/state/src/plugin/mod.rs @@ -196,8 +196,14 @@ impl PluginMgr { .unwrap_or(false) { info!("Loading plugin at {:?}", entry.path()); - Plugin::from_reader(fs::File::open(entry.path()).map_err(PluginError::Io)?) - .map(Some) + Plugin::from_reader(fs::File::open(entry.path()).map_err(PluginError::Io)?).map( + |o| { + if let Err(e) = common::assets::register_tar(entry.path()) { + error!("Plugin {:?} tar error {e:?}", entry.path()); + } + Some(o) + }, + ) } else { Ok(None) } diff --git a/voxygen/Cargo.toml b/voxygen/Cargo.toml index 3425feca58..ef7c7a4bde 100644 --- a/voxygen/Cargo.toml +++ b/voxygen/Cargo.toml @@ -27,7 +27,7 @@ singleplayer = ["server"] simd = ["vek/platform_intrinsics"] tracy = ["common-frontend/tracy", "client/tracy"] tracy-memory = ["tracy"] # enables heap profiling with tracy -plugins = ["client/plugins"] +plugins = ["client/plugins", "common-assets/plugins", "common-assets/hashbrown"] egui-ui = ["voxygen-egui", "egui", "egui_wgpu_backend", "egui_winit_platform"] shaderc-from-source = ["shaderc/build-from-source"] discord = ["discord-sdk"] diff --git a/voxygen/src/scene/figure/load.rs b/voxygen/src/scene/figure/load.rs index 5cabb93f1b..f17d166515 100644 --- a/voxygen/src/scene/figure/load.rs +++ b/voxygen/src/scene/figure/load.rs @@ -2,7 +2,7 @@ use super::cache::{ FigureKey, FigureModelEntryFuture, ModelEntryFuture, TerrainModelEntryFuture, ToolKey, }; use common::{ - assets::{self, AssetExt, AssetHandle, DotVoxAsset, ReloadWatcher, Ron}, + assets::{self, AssetExt, AssetHandle, Concatenate, DotVoxAsset, MultiRon, ReloadWatcher}, comp::{ arthropod::{self, BodyType as ABodyType, Species as ASpecies}, biped_large::{self, BodyType as BLBodyType, Species as BLSpecies}, @@ -143,7 +143,7 @@ macro_rules! make_vox_spec { ) => { #[derive(Clone)] pub struct $Spec { - $( $field: AssetHandle>, )* + $( $field: AssetHandle>, )* } impl assets::Compound for $Spec { @@ -178,6 +178,13 @@ macro_rules! make_vox_spec { } } } +macro_rules! concatenate_tuple { + ($name:ty) => { + impl Concatenate for $name { + fn concatenate(self, b: Self) -> Self { Self(self.0.concatenate(b.0)) } + } + }; +} // All offsets should be relative to an initial origin that doesn't change when // combining segments @@ -259,6 +266,10 @@ impl HumColorSpec { } } +impl Concatenate for HumColorSpec { + fn concatenate(self, _b: Self) -> Self { todo!("Can't concatenate HumColorSpec") } +} + // All reliant on humanoid::Species and humanoid::BodyType #[derive(Deserialize)] struct HumHeadSubSpec { @@ -364,6 +375,7 @@ impl HumHeadSpec { ) } } +concatenate_tuple!(HumHeadSpec); // Armor aspects should be in the same order, top to bottom. // These seem overly split up, but wanted to keep the armor seperated @@ -376,32 +388,53 @@ where default: S, map: HashMap, } +impl Concatenate for ArmorVoxSpecMap { + fn concatenate(self, b: Self) -> Self { + Self { + default: self.default, + map: self.map.concatenate(b.map), + } + } +} #[derive(Deserialize)] struct HumArmorShoulderSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorShoulderSpec); #[derive(Deserialize)] struct HumArmorChestSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorChestSpec); #[derive(Deserialize)] struct HumArmorHandSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorHandSpec); #[derive(Deserialize)] struct HumArmorBeltSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorBeltSpec); #[derive(Deserialize)] struct HumArmorBackSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorBackSpec); #[derive(Deserialize)] struct HumArmorPantsSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorPantsSpec); #[derive(Deserialize)] struct HumArmorFootSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorFootSpec); #[derive(Deserialize)] struct HumMainWeaponSpec(HashMap); +concatenate_tuple!(HumMainWeaponSpec); #[derive(Deserialize)] struct HumModularComponentSpec(HashMap); +concatenate_tuple!(HumModularComponentSpec); #[derive(Deserialize)] struct HumArmorLanternSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorLanternSpec); #[derive(Deserialize)] struct HumArmorGliderSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorGliderSpec); #[derive(Deserialize)] struct HumArmorHeadSpec(ArmorVoxSpecMap<(Species, BodyType, String), ArmorVoxSpec>); +concatenate_tuple!(HumArmorHeadSpec); #[derive(Deserialize)] struct HumArmorTabardSpec(ArmorVoxSpecMap); +concatenate_tuple!(HumArmorTabardSpec); make_vox_spec!( Body, @@ -1072,6 +1105,7 @@ fn mesh_hold() -> BoneMeshes { ////// #[derive(Deserialize)] struct QuadrupedSmallCentralSpec(HashMap<(QSSpecies, QSBodyType), SidedQSCentralVoxSpec>); +concatenate_tuple!(QuadrupedSmallCentralSpec); #[derive(Deserialize)] struct SidedQSCentralVoxSpec { @@ -1089,6 +1123,7 @@ struct QuadrupedSmallCentralSubSpec { #[derive(Deserialize)] struct QuadrupedSmallLateralSpec(HashMap<(QSSpecies, QSBodyType), SidedQSLateralVoxSpec>); +concatenate_tuple!(QuadrupedSmallLateralSpec); #[derive(Deserialize)] struct SidedQSLateralVoxSpec { @@ -1287,6 +1322,7 @@ impl QuadrupedSmallLateralSpec { ////// #[derive(Deserialize)] struct QuadrupedMediumCentralSpec(HashMap<(QMSpecies, QMBodyType), SidedQMCentralVoxSpec>); +concatenate_tuple!(QuadrupedMediumCentralSpec); #[derive(Deserialize)] struct SidedQMCentralVoxSpec { @@ -1308,6 +1344,7 @@ struct QuadrupedMediumCentralSubSpec { #[derive(Deserialize)] struct QuadrupedMediumLateralSpec(HashMap<(QMSpecies, QMBodyType), SidedQMLateralVoxSpec>); +concatenate_tuple!(QuadrupedMediumLateralSpec); #[derive(Deserialize)] struct SidedQMLateralVoxSpec { leg_fl: QuadrupedMediumLateralSubSpec, @@ -1663,6 +1700,7 @@ impl QuadrupedMediumLateralSpec { ////// #[derive(Deserialize)] struct BirdMediumCentralSpec(HashMap<(BMSpecies, BMBodyType), SidedBMCentralVoxSpec>); +concatenate_tuple!(BirdMediumCentralSpec); #[derive(Deserialize)] struct SidedBMCentralVoxSpec { @@ -1680,6 +1718,7 @@ struct BirdMediumCentralSubSpec { #[derive(Deserialize)] struct BirdMediumLateralSpec(HashMap<(BMSpecies, BMBodyType), SidedBMLateralVoxSpec>); +concatenate_tuple!(BirdMediumLateralSpec); #[derive(Deserialize)] struct SidedBMLateralVoxSpec { @@ -1917,6 +1956,7 @@ impl BirdMediumLateralSpec { ////// #[derive(Deserialize)] struct TheropodCentralSpec(HashMap<(TSpecies, TBodyType), SidedTCentralVoxSpec>); +concatenate_tuple!(TheropodCentralSpec); #[derive(Deserialize)] struct SidedTCentralVoxSpec { @@ -1937,6 +1977,7 @@ struct TheropodCentralSubSpec { } #[derive(Deserialize)] struct TheropodLateralSpec(HashMap<(TSpecies, TBodyType), SidedTLateralVoxSpec>); +concatenate_tuple!(TheropodLateralSpec); #[derive(Deserialize)] struct SidedTLateralVoxSpec { @@ -2247,6 +2288,7 @@ impl TheropodLateralSpec { ////// #[derive(Deserialize)] struct ArthropodCentralSpec(HashMap<(ASpecies, ABodyType), SidedACentralVoxSpec>); +concatenate_tuple!(ArthropodCentralSpec); #[derive(Deserialize)] struct SidedACentralVoxSpec { @@ -2262,6 +2304,7 @@ struct ArthropodCentralSubSpec { } #[derive(Deserialize)] struct ArthropodLateralSpec(HashMap<(ASpecies, ABodyType), SidedALateralVoxSpec>); +concatenate_tuple!(ArthropodLateralSpec); #[derive(Deserialize)] struct SidedALateralVoxSpec { @@ -2647,6 +2690,7 @@ impl ArthropodLateralSpec { ////// #[derive(Deserialize)] struct FishMediumCentralSpec(HashMap<(FMSpecies, FMBodyType), SidedFMCentralVoxSpec>); +concatenate_tuple!(FishMediumCentralSpec); #[derive(Deserialize)] struct SidedFMCentralVoxSpec { @@ -2665,6 +2709,7 @@ struct FishMediumCentralSubSpec { } #[derive(Deserialize)] struct FishMediumLateralSpec(HashMap<(FMSpecies, FMBodyType), SidedFMLateralVoxSpec>); +concatenate_tuple!(FishMediumLateralSpec); #[derive(Deserialize)] struct SidedFMLateralVoxSpec { fin_l: FishMediumLateralSubSpec, @@ -2853,6 +2898,7 @@ impl FishMediumLateralSpec { ////// #[derive(Deserialize)] struct FishSmallCentralSpec(HashMap<(FSSpecies, FSBodyType), SidedFSCentralVoxSpec>); +concatenate_tuple!(FishSmallCentralSpec); #[derive(Deserialize)] struct SidedFSCentralVoxSpec { @@ -2868,6 +2914,7 @@ struct FishSmallCentralSubSpec { } #[derive(Deserialize)] struct FishSmallLateralSpec(HashMap<(FSSpecies, FSBodyType), SidedFSLateralVoxSpec>); +concatenate_tuple!(FishSmallLateralSpec); #[derive(Deserialize)] struct SidedFSLateralVoxSpec { fin_l: FishSmallLateralSubSpec, @@ -2998,18 +3045,25 @@ impl FishSmallLateralSpec { #[derive(Deserialize)] struct BipedSmallWeaponSpec(HashMap); +concatenate_tuple!(BipedSmallWeaponSpec); #[derive(Deserialize)] struct BipedSmallArmorHeadSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorHeadSpec); #[derive(Deserialize)] struct BipedSmallArmorHandSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorHandSpec); #[derive(Deserialize)] struct BipedSmallArmorFootSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorFootSpec); #[derive(Deserialize)] struct BipedSmallArmorChestSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorChestSpec); #[derive(Deserialize)] struct BipedSmallArmorPantsSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorPantsSpec); #[derive(Deserialize)] struct BipedSmallArmorTailSpec(ArmorVoxSpecMap); +concatenate_tuple!(BipedSmallArmorTailSpec); make_vox_spec!( biped_small::Body, struct BipedSmallSpec { @@ -3273,6 +3327,7 @@ impl BipedSmallWeaponSpec { ////// #[derive(Deserialize)] struct DragonCentralSpec(HashMap<(DSpecies, DBodyType), SidedDCentralVoxSpec>); +concatenate_tuple!(DragonCentralSpec); #[derive(Deserialize)] struct SidedDCentralVoxSpec { @@ -3294,6 +3349,7 @@ struct DragonCentralSubSpec { #[derive(Deserialize)] struct DragonLateralSpec(HashMap<(DSpecies, DBodyType), SidedDLateralVoxSpec>); +concatenate_tuple!(DragonLateralSpec); #[derive(Deserialize)] struct SidedDLateralVoxSpec { @@ -3644,6 +3700,7 @@ impl DragonLateralSpec { ////// #[derive(Deserialize)] struct BirdLargeCentralSpec(HashMap<(BLASpecies, BLABodyType), SidedBLACentralVoxSpec>); +concatenate_tuple!(BirdLargeCentralSpec); #[derive(Deserialize)] struct SidedBLACentralVoxSpec { @@ -3664,6 +3721,7 @@ struct BirdLargeCentralSubSpec { #[derive(Deserialize)] struct BirdLargeLateralSpec(HashMap<(BLASpecies, BLABodyType), SidedBLALateralVoxSpec>); +concatenate_tuple!(BirdLargeLateralSpec); #[derive(Deserialize)] struct SidedBLALateralVoxSpec { @@ -4047,6 +4105,7 @@ impl BirdLargeLateralSpec { ////// #[derive(Deserialize)] struct BipedLargeCentralSpec(HashMap<(BLSpecies, BLBodyType), SidedBLCentralVoxSpec>); +concatenate_tuple!(BipedLargeCentralSpec); #[derive(Deserialize)] struct SidedBLCentralVoxSpec { @@ -4066,6 +4125,7 @@ struct BipedLargeCentralSubSpec { #[derive(Deserialize)] struct BipedLargeLateralSpec(HashMap<(BLSpecies, BLBodyType), SidedBLLateralVoxSpec>); +concatenate_tuple!(BipedLargeLateralSpec); #[derive(Deserialize)] struct SidedBLLateralVoxSpec { @@ -4087,8 +4147,10 @@ struct BipedLargeLateralSubSpec { } #[derive(Deserialize)] struct BipedLargeMainSpec(HashMap); +concatenate_tuple!(BipedLargeMainSpec); #[derive(Deserialize)] struct BipedLargeSecondSpec(HashMap); +concatenate_tuple!(BipedLargeSecondSpec); make_vox_spec!( biped_large::Body, struct BipedLargeSpec { @@ -4466,6 +4528,7 @@ impl BipedLargeSecondSpec { ////// #[derive(Deserialize)] struct GolemCentralSpec(HashMap<(GSpecies, GBodyType), SidedGCentralVoxSpec>); +concatenate_tuple!(GolemCentralSpec); #[derive(Deserialize)] struct SidedGCentralVoxSpec { @@ -4484,6 +4547,7 @@ struct GolemCentralSubSpec { #[derive(Deserialize)] struct GolemLateralSpec(HashMap<(GSpecies, GBodyType), SidedGLateralVoxSpec>); +concatenate_tuple!(GolemLateralSpec); #[derive(Deserialize)] struct SidedGLateralVoxSpec { @@ -4776,6 +4840,7 @@ impl GolemLateralSpec { ////// #[derive(Deserialize)] struct QuadrupedLowCentralSpec(HashMap<(QLSpecies, QLBodyType), SidedQLCentralVoxSpec>); +concatenate_tuple!(QuadrupedLowCentralSpec); #[derive(Deserialize)] struct SidedQLCentralVoxSpec { @@ -4796,6 +4861,7 @@ struct QuadrupedLowCentralSubSpec { #[derive(Deserialize)] struct QuadrupedLowLateralSpec(HashMap<(QLSpecies, QLBodyType), SidedQLLateralVoxSpec>); +concatenate_tuple!(QuadrupedLowLateralSpec); #[derive(Deserialize)] struct SidedQLLateralVoxSpec { front_left: QuadrupedLowLateralSubSpec, @@ -5125,6 +5191,7 @@ impl ObjectCentralSpec { (central, Vec3::from(spec.bone1.offset)) } } +concatenate_tuple!(ObjectCentralSpec); struct ModelWithOptionalIndex(String, u32); @@ -5162,6 +5229,7 @@ impl<'de> Deserialize<'de> for ModelWithOptionalIndex { #[derive(Deserialize)] struct ItemDropCentralSpec(HashMap); +concatenate_tuple!(ItemDropCentralSpec); make_vox_spec!( item_drop::Body, From 3a6d94bd3cd801026c804edc8e1978d19ffe86be Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sat, 14 Oct 2023 23:07:31 +0200 Subject: [PATCH 02/11] register_tar with the cache object --- common/assets/src/lib.rs | 4 +++- common/assets/src/plugin_cache.rs | 16 ++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/common/assets/src/lib.rs b/common/assets/src/lib.rs index abc3e659ef..1364d355e8 100644 --- a/common/assets/src/lib.rs +++ b/common/assets/src/lib.rs @@ -25,7 +25,6 @@ mod fs; mod plugin_cache; #[cfg(feature = "plugins")] mod tar_source; mod walk; -pub use plugin_cache::register_tar; pub use walk::{walk_tree, Walk}; lazy_static! { @@ -36,6 +35,9 @@ lazy_static! { #[cfg(feature = "hot-reloading")] pub fn start_hot_reloading() { ASSETS.enhance_hot_reloading(); } +// register a new plugin +pub fn register_tar(path: PathBuf) -> std::io::Result<()> { ASSETS.register_tar(path) } + pub type AssetHandle = assets_manager::Handle<'static, T>; pub type AssetGuard = assets_manager::AssetGuard<'static, T>; pub type AssetDirHandle = assets_manager::DirHandle<'static, T>; diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index e626dbdba2..8cb3ab8c4a 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -18,14 +18,6 @@ lazy_static! { static ref PLUGIN_LIST: RwLock> = RwLock::new(Vec::new()); } -pub fn register_tar(path: PathBuf) -> Result<(), Box> { - let tar_source = Tar::from_path(&path)?; - println!("Tar {:?} {:?}", path, tar_source); - let cache = AssetCache::with_source(tar_source); - PLUGIN_LIST.write()?.push(PluginEntry { path, cache }); - Ok(()) -} - /// The source combining filesystem and plugins (typically used via /// CombinedCache) #[derive(Debug, Clone)] @@ -139,6 +131,14 @@ impl CombinedCache { } result } + + pub fn register_tar(&self, path: PathBuf) -> std::io::Result<()> { + let tar_source = Tar::from_path(&path)?; + println!("Tar {:?} {:?}", path, tar_source); + let cache = AssetCache::with_source(tar_source); + PLUGIN_LIST.write().unwrap().push(PluginEntry { path, cache }); + Ok(()) + } } impl std::ops::Deref for CombinedCache { From baccdeb22fc375b3157da48de6b10f520e60c139 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 00:13:57 +0200 Subject: [PATCH 03/11] move plugin list into source --- common/assets/src/plugin_cache.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 8cb3ab8c4a..0f1571d5bc 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -5,30 +5,26 @@ use crate::Concatenate; use super::{fs::FileSystem, tar_source::Tar}; use assets_manager::{ source::{FileContent, Source}, - AnyCache, AssetCache, BoxedError, + AnyCache, AssetCache, BoxedError, hot_reloading::{EventSender, DynUpdateSender}, }; -use lazy_static::lazy_static; struct PluginEntry { path: PathBuf, cache: AssetCache, } -lazy_static! { - static ref PLUGIN_LIST: RwLock> = RwLock::new(Vec::new()); -} - /// The source combining filesystem and plugins (typically used via /// CombinedCache) -#[derive(Debug, Clone)] pub struct CombinedSource { fs: FileSystem, + plugin_list: RwLock>, } impl CombinedSource { pub fn new() -> std::io::Result { Ok(Self { fs: FileSystem::new()?, + plugin_list: RwLock::new(Vec::new()), }) } } @@ -39,7 +35,7 @@ impl CombinedSource { if let Ok(file_entry) = self.fs.read(id, ext) { result.push((None, file_entry)); } - if let Ok(guard) = PLUGIN_LIST.read() { + if let Ok(guard) = self.plugin_list.read() { for (n, p) in guard.iter().enumerate() { if let Ok(entry) = p.cache.raw_source().read(id, ext) { result.push((Some(n), match entry { @@ -58,7 +54,7 @@ impl CombinedSource { // we don't want to keep the lock, so we clone fn plugin_path(&self, index: Option) -> Option { if let Some(index) = index { - PLUGIN_LIST + self.plugin_list .read() .ok() .and_then(|p| p.get(index).map(|p| p.path.clone())) @@ -97,13 +93,18 @@ impl Source for CombinedSource { fn exists(&self, entry: assets_manager::source::DirEntry) -> bool { self.fs.exists(entry) - || PLUGIN_LIST + || self.plugin_list .read() .map(|p| p.iter().any(|p| p.cache.raw_source().exists(entry))) .unwrap_or_default() } - fn make_source(&self) -> Option> { None } + // TODO: Enable hot reloading for plugins + fn make_source(&self) -> Option> { self.fs.make_source() } + + fn configure_hot_reloading(&self, events: EventSender) -> Result { + self.fs.configure_hot_reloading(events) + } } /// A cache combining filesystem and plugin assets @@ -120,7 +121,7 @@ impl CombinedCache { load_from: impl Fn(AnyCache) -> Result, ) -> Result { let mut result = load_from(self.as_any_cache()); - for i in PLUGIN_LIST.read().unwrap().iter() { + for i in self.0.raw_source().plugin_list.read().unwrap().iter() { if let Ok(b) = load_from(i.cache.as_any_cache()) { result = if let Ok(a) = result { Ok(a.concatenate(b)) @@ -136,7 +137,7 @@ impl CombinedCache { let tar_source = Tar::from_path(&path)?; println!("Tar {:?} {:?}", path, tar_source); let cache = AssetCache::with_source(tar_source); - PLUGIN_LIST.write().unwrap().push(PluginEntry { path, cache }); + self.0.raw_source().plugin_list.write().unwrap().push(PluginEntry { path, cache }); Ok(()) } } From 8dce52221b7cbb12d6d2ca9ce87e5e54cdd4f810 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 00:22:29 +0200 Subject: [PATCH 04/11] separate combined cache and fs cache --- common/assets/src/plugin_cache.rs | 33 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 0f1571d5bc..04361fc1c4 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -4,8 +4,9 @@ use crate::Concatenate; use super::{fs::FileSystem, tar_source::Tar}; use assets_manager::{ + hot_reloading::{DynUpdateSender, EventSender}, source::{FileContent, Source}, - AnyCache, AssetCache, BoxedError, hot_reloading::{EventSender, DynUpdateSender}, + AnyCache, AssetCache, BoxedError, }; struct PluginEntry { @@ -16,14 +17,14 @@ struct PluginEntry { /// The source combining filesystem and plugins (typically used via /// CombinedCache) pub struct CombinedSource { - fs: FileSystem, + fs: AssetCache, plugin_list: RwLock>, } impl CombinedSource { pub fn new() -> std::io::Result { Ok(Self { - fs: FileSystem::new()?, + fs: AssetCache::with_source(FileSystem::new()?), plugin_list: RwLock::new(Vec::new()), }) } @@ -32,7 +33,7 @@ impl CombinedSource { impl CombinedSource { fn read_multiple(&self, id: &str, ext: &str) -> Vec<(Option, FileContent<'_>)> { let mut result = Vec::new(); - if let Ok(file_entry) = self.fs.read(id, ext) { + if let Ok(file_entry) = self.fs.raw_source().read(id, ext) { result.push((None, file_entry)); } if let Ok(guard) = self.plugin_list.read() { @@ -88,22 +89,23 @@ impl Source for CombinedSource { f: &mut dyn FnMut(assets_manager::source::DirEntry), ) -> std::io::Result<()> { // TODO: we should combine the sources - self.fs.read_dir(id, f) + self.fs.raw_source().read_dir(id, f) } fn exists(&self, entry: assets_manager::source::DirEntry) -> bool { - self.fs.exists(entry) - || self.plugin_list + self.fs.raw_source().exists(entry) + || self + .plugin_list .read() .map(|p| p.iter().any(|p| p.cache.raw_source().exists(entry))) .unwrap_or_default() } // TODO: Enable hot reloading for plugins - fn make_source(&self) -> Option> { self.fs.make_source() } + fn make_source(&self) -> Option> { self.fs.raw_source().make_source() } fn configure_hot_reloading(&self, events: EventSender) -> Result { - self.fs.configure_hot_reloading(events) + self.fs.raw_source().configure_hot_reloading(events) } } @@ -120,7 +122,7 @@ impl CombinedCache { &self, load_from: impl Fn(AnyCache) -> Result, ) -> Result { - let mut result = load_from(self.as_any_cache()); + let mut result = load_from(self.0.raw_source().fs.as_any_cache()); for i in self.0.raw_source().plugin_list.read().unwrap().iter() { if let Ok(b) = load_from(i.cache.as_any_cache()) { result = if let Ok(a) = result { @@ -135,11 +137,16 @@ impl CombinedCache { pub fn register_tar(&self, path: PathBuf) -> std::io::Result<()> { let tar_source = Tar::from_path(&path)?; - println!("Tar {:?} {:?}", path, tar_source); + //println!("Tar {:?} {:?}", path, tar_source); let cache = AssetCache::with_source(tar_source); - self.0.raw_source().plugin_list.write().unwrap().push(PluginEntry { path, cache }); + self.0 + .raw_source() + .plugin_list + .write() + .unwrap() + .push(PluginEntry { path, cache }); Ok(()) - } + } } impl std::ops::Deref for CombinedCache { From 61b7a1ff177f6ead5b5799170bf26341e86e33b6 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 19:06:17 +0200 Subject: [PATCH 05/11] report errors other than not found when combining assets --- common/assets/src/plugin_cache.rs | 41 ++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 04361fc1c4..81bae823af 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -39,6 +39,7 @@ impl CombinedSource { if let Ok(guard) = self.plugin_list.read() { for (n, p) in guard.iter().enumerate() { if let Ok(entry) = p.cache.raw_source().read(id, ext) { + // the data is behind an RwLockReadGuard, so own it for returning result.push((Some(n), match entry { FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), FileContent::Buffer(b) => FileContent::Buffer(b), @@ -123,13 +124,41 @@ impl CombinedCache { load_from: impl Fn(AnyCache) -> Result, ) -> Result { let mut result = load_from(self.0.raw_source().fs.as_any_cache()); + // report a severe error from the filesystem asset even if later overwritten by + // an Ok value from a plugin + match result { + Err(ref e) => { + match e + .source() + .and_then(|s| s.downcast_ref::()) + .map(|e| e.kind()) + { + Some(std::io::ErrorKind::NotFound) => (), + _ => tracing::error!("Filesystem asset load {e:?}"), + } + }, + _ => (), + } for i in self.0.raw_source().plugin_list.read().unwrap().iter() { - if let Ok(b) = load_from(i.cache.as_any_cache()) { - result = if let Ok(a) = result { - Ok(a.concatenate(b)) - } else { - Ok(b) - }; + match load_from(i.cache.as_any_cache()) { + Ok(b) => { + result = if let Ok(a) = result { + Ok(a.concatenate(b)) + } else { + Ok(b) + }; + }, + // report any error other than NotFound + Err(e) => { + match e + .source() + .and_then(|s| s.downcast_ref::()) + .map(|e| e.kind()) + { + Some(std::io::ErrorKind::NotFound) => (), + _ => tracing::error!("Loading from {:?} failed {e:?}", i.path), + } + }, } } result From 60336a1c568728b556ac063db9a6b442213fffb0 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 19:30:37 +0200 Subject: [PATCH 06/11] improve duplicate error message --- common/assets/src/plugin_cache.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 81bae823af..1972ddb71f 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -2,7 +2,7 @@ use std::{path::PathBuf, sync::RwLock}; use crate::Concatenate; -use super::{fs::FileSystem, tar_source::Tar}; +use super::{fs::FileSystem, tar_source::Tar, ASSETS_PATH}; use assets_manager::{ hot_reloading::{DynUpdateSender, EventSender}, source::{FileContent, Source}, @@ -74,11 +74,11 @@ impl Source for CombinedSource { Err(std::io::ErrorKind::NotFound.into()) } else { if entries.len() > 1 { - tracing::error!( - "Duplicate asset {id} in read, plugins {:?} + {:?}", - self.plugin_path(entries[0].0), - self.plugin_path(entries[1].0) - ); + let plugina = self.plugin_path(entries[0].0); + let pluginb = self.plugin_path(entries[1].0); + let patha = plugina.as_ref().unwrap_or(&ASSETS_PATH); + let pathb = pluginb.as_ref().unwrap_or(&ASSETS_PATH); + tracing::error!("Duplicate asset {id} in {patha:?} and {pathb:?}"); } Ok(entries.swap_remove(0).1) } From 73ebe8ab987d8edf18f867efc9defc279fe8b235 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 19:43:39 +0200 Subject: [PATCH 07/11] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d62fa4d3..a56c9ac58a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New arena building in desert cities, suitable for PVP, also NPCs like to watch the fights too - The loading screen now displays status updates for singleplayer server and client initialization progress - New Frost Gigas attacks & AI +- Weapons and armor load from plugins ### Changed From 64dec7efe22c02e66a9955518415741cbbeed06b Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 21:01:15 +0200 Subject: [PATCH 08/11] fix compilation without plugins feature, fixes unit tests --- common/assets/src/lib.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/common/assets/src/lib.rs b/common/assets/src/lib.rs index 1364d355e8..f19ecf36a8 100644 --- a/common/assets/src/lib.rs +++ b/common/assets/src/lib.rs @@ -22,20 +22,28 @@ pub use assets_manager::{ }; mod fs; -mod plugin_cache; +#[cfg(feature = "plugins")] mod plugin_cache; #[cfg(feature = "plugins")] mod tar_source; mod walk; pub use walk::{walk_tree, Walk}; +#[cfg(feature = "plugins")] lazy_static! { - /// The HashMap where all loaded assets are stored in. - static ref ASSETS: plugin_cache::CombinedCache = plugin_cache::CombinedCache::new().unwrap(); +/// The HashMap where all loaded assets are stored in. +static ref ASSETS: plugin_cache::CombinedCache = plugin_cache::CombinedCache::new().unwrap(); +} +#[cfg(not(feature = "plugins"))] +lazy_static! { +/// The HashMap where all loaded assets are stored in. +static ref ASSETS: AssetCache = + AssetCache::with_source(fs::FileSystem::new().unwrap()); } #[cfg(feature = "hot-reloading")] pub fn start_hot_reloading() { ASSETS.enhance_hot_reloading(); } // register a new plugin +#[cfg(feature = "plugins")] pub fn register_tar(path: PathBuf) -> std::io::Result<()> { ASSETS.register_tar(path) } pub type AssetHandle = assets_manager::Handle<'static, T>; @@ -258,6 +266,7 @@ impl Clone for MultiRon { fn clone_from(&mut self, source: &Self) { self.0.clone_from(&source.0) } } +#[cfg(feature = "plugins")] impl Compound for MultiRon where T: for<'de> serde::Deserialize<'de> + Send + Sync + 'static + Concatenate, From 621334d69bc5e8877b5649535da48c81066e1951 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 21:54:27 +0200 Subject: [PATCH 09/11] clippy fix and tar_source simplification --- common/assets/src/plugin_cache.rs | 21 ++--- common/assets/src/tar_source.rs | 145 ++++++++---------------------- 2 files changed, 46 insertions(+), 120 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 1972ddb71f..595eb9995b 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -126,18 +126,15 @@ impl CombinedCache { let mut result = load_from(self.0.raw_source().fs.as_any_cache()); // report a severe error from the filesystem asset even if later overwritten by // an Ok value from a plugin - match result { - Err(ref e) => { - match e - .source() - .and_then(|s| s.downcast_ref::()) - .map(|e| e.kind()) - { - Some(std::io::ErrorKind::NotFound) => (), - _ => tracing::error!("Filesystem asset load {e:?}"), - } - }, - _ => (), + if let Err(ref e) = result { + match e + .source() + .and_then(|s| s.downcast_ref::()) + .map(|e| e.kind()) + { + Some(std::io::ErrorKind::NotFound) => (), + _ => tracing::error!("Filesystem asset load {e:?}"), + } } for i in self.0.raw_source().plugin_list.read().unwrap().iter() { match load_from(i.cache.as_any_cache()) { diff --git a/common/assets/src/tar_source.rs b/common/assets/src/tar_source.rs index 259d184f42..8e2c86e914 100644 --- a/common/assets/src/tar_source.rs +++ b/common/assets/src/tar_source.rs @@ -10,15 +10,7 @@ use std::{ path::{self, Path, PathBuf}, }; -// derived from the zip source from assets_manager - -#[inline] -pub(crate) fn extension_of(path: &Path) -> Option<&str> { - match path.extension() { - Some(ext) => ext.to_str(), - None => Some(""), - } -} +// derived from the zip source in the assets_manager crate #[derive(Clone, Hash, PartialEq, Eq)] struct FileDesc(String, String); @@ -68,126 +60,74 @@ impl fmt::Debug for FileDesc { } } -/// An entry in a archive directory. -#[derive(Debug)] -enum OwnedEntry { - File(FileDesc), - // Dir(String), +impl FileDesc { + fn as_dir_entry(&self) -> DirEntry { DirEntry::File(&self.0, &self.1) } } -impl OwnedEntry { - fn as_dir_entry(&self) -> DirEntry { - match self { - OwnedEntry::File(FileDesc(desc0, desc1)) => DirEntry::File(desc0, desc1), - // OwnedEntry::Dir(id) => DirEntry::Directory(id), - } - } -} - -/// Build ids from components. -/// -/// Using this allows to easily reuse buffers when building several ids in a -/// row, and thus to avoid repeated allocations. -#[derive(Default)] -struct IdBuilder { - segments: Vec, - len: usize, -} - -impl IdBuilder { - /// Pushs a segment in the builder. - #[inline] - fn push(&mut self, s: &str) { - match self.segments.get_mut(self.len) { - Some(seg) => { - seg.clear(); - seg.push_str(s); - }, - None => self.segments.push(s.to_owned()), - } - self.len += 1; - } - - /// Joins segments to build a id. - #[inline] - fn join(&self) -> String { self.segments[..self.len].join(".") } - - /// Resets the builder without freeing buffers. - #[inline] - fn reset(&mut self) { self.len = 0; } -} - -/// Register a file of an archive in maps. +/// Register a file of an archive in maps, components in asset ids are separated +/// by points fn register_file( path: &Path, - position: usize, + position: u64, length: usize, - files: &mut HashMap, - dirs: &mut HashMap>, - id_builder: &mut IdBuilder, + files: &mut HashMap, + dirs: &mut HashMap>, ) { - id_builder.reset(); - // Parse the path and register it. + let mut parent_id = String::default(); // The closure is used as a cheap `try` block. - let ok = (|| { - // Fill `id_builder` from the parent's components + let unsupported_path = (|| { let parent = path.parent()?; for comp in parent.components() { match comp { path::Component::Normal(s) => { let segment = s.to_str()?; + // reject paths with extensions if segment.contains('.') { return None; } - id_builder.push(segment); + if !parent_id.is_empty() { + parent_id.push('.'); + } + parent_id.push_str(segment); }, + // reject paths with non-name components _ => return None, } } - // Build the ids of the file and its parent. - let parent_id = id_builder.join(); - id_builder.push(path.file_stem()?.to_str()?); - let id = id_builder.join(); - + let file_id = parent_id.clone() + "." + path.file_stem()?.to_str()?; // Register the file in the maps. - let ext = extension_of(path)?.to_owned(); - let desc = FileDesc(id, ext); + let ext = path.extension().unwrap_or_default().to_str()?.to_owned(); + let desc = FileDesc(file_id, ext); files.insert(desc.clone(), (position, length)); - let entry = OwnedEntry::File(desc); - dirs.entry(parent_id).or_default().push(entry); + dirs.entry(parent_id).or_default().push(desc); Some(()) })() - .is_some(); - - if !ok { + .is_none(); + if unsupported_path { tracing::warn!("Unsupported path in tar archive: {path:?}"); } } -enum Backend { - File(PathBuf), - // Buffer(&'static [u8]), -} +// we avoid the extra dependency of sync_file introduced by Zip here by opening +// the file for each read +struct Backend(PathBuf); impl Backend { - fn read(&self, pos: usize, len: usize) -> std::io::Result> { - match self { - Backend::File(path) => File::open(path).and_then(|file| { - let mut result = vec![0; len]; - file.read_at(result.as_mut_slice(), pos as u64) - .map(move |_bytes| result) - }), - // Backend::Buffer(_) => todo!(), - } + fn read(&self, pos: u64, len: usize) -> std::io::Result> { + File::open(self.0.clone()).and_then(|file| { + let mut result = vec![0; len]; + file.read_at(result.as_mut_slice(), pos) + .map(move |_bytes| result) + }) } } pub struct Tar { - files: HashMap, - dirs: HashMap>, + files: HashMap, + dirs: HashMap>, backend: Backend, } @@ -201,23 +141,21 @@ impl Tar { .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; let mut files = HashMap::with_capacity(contents.size_hint().0); let mut dirs = HashMap::new(); - let mut id_builder = IdBuilder::default(); for e in contents.flatten() { if matches!(e.header().entry_type(), EntryType::Regular) { register_file( e.path().map_err(io::Error::other)?.as_ref(), - e.raw_file_position() as usize, + e.raw_file_position(), e.size() as usize, &mut files, &mut dirs, - &mut id_builder, ); } } Ok(Tar { files, dirs, - backend: Backend::File(path.to_path_buf()), + backend: Backend(path.to_path_buf()), }) } } @@ -225,22 +163,13 @@ impl Tar { impl Source for Tar { fn read(&self, id: &str, ext: &str) -> io::Result { let key: &dyn FileKey = &(id, ext); - let id = *self - .files - .get(key) - .or_else(|| { - // also accept assets within the assets dir for now - let with_prefix = "assets.".to_string() + id; - let prefixed_key: &dyn FileKey = &(with_prefix.as_str(), ext); - self.files.get(prefixed_key) - }) - .ok_or(io::ErrorKind::NotFound)?; + let id = *self.files.get(key).ok_or(io::ErrorKind::NotFound)?; self.backend.read(id.0, id.1).map(FileContent::Buffer) } fn read_dir(&self, id: &str, f: &mut dyn FnMut(DirEntry)) -> io::Result<()> { let dir = self.dirs.get(id).ok_or(io::ErrorKind::NotFound)?; - dir.iter().map(OwnedEntry::as_dir_entry).for_each(f); + dir.iter().map(FileDesc::as_dir_entry).for_each(f); Ok(()) } From 22f8433d2289c36a6407b811472dae76080f5fa3 Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Sun, 15 Oct 2023 23:34:05 +0200 Subject: [PATCH 10/11] address imbris suggestions in this new version --- common/assets/src/plugin_cache.rs | 72 +++++++++++++++---------------- common/assets/src/tar_source.rs | 22 +++++----- common/state/src/plugin/mod.rs | 4 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 595eb9995b..261d17e981 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -36,30 +36,27 @@ impl CombinedSource { if let Ok(file_entry) = self.fs.raw_source().read(id, ext) { result.push((None, file_entry)); } - if let Ok(guard) = self.plugin_list.read() { - for (n, p) in guard.iter().enumerate() { - if let Ok(entry) = p.cache.raw_source().read(id, ext) { - // the data is behind an RwLockReadGuard, so own it for returning - result.push((Some(n), match entry { - FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), - FileContent::Buffer(b) => FileContent::Buffer(b), - FileContent::Owned(s) => { - FileContent::Buffer(Vec::from(s.as_ref().as_ref())) - }, - })); - } + for (n, p) in self.plugin_list.read().unwrap().iter().enumerate() { + if let Ok(entry) = p.cache.raw_source().read(id, ext) { + // the data is behind an RwLockReadGuard, so own it for returning + result.push((Some(n), match entry { + FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), + FileContent::Buffer(b) => FileContent::Buffer(b), + FileContent::Owned(s) => FileContent::Buffer(Vec::from(s.as_ref().as_ref())), + })); } } result } - // we don't want to keep the lock, so we clone + // We don't want to keep the lock, so we clone fn plugin_path(&self, index: Option) -> Option { if let Some(index) = index { self.plugin_list .read() - .ok() - .and_then(|p| p.get(index).map(|p| p.path.clone())) + .unwrap() + .get(index) + .map(|plugin| plugin.path.clone()) } else { None } @@ -68,7 +65,7 @@ impl CombinedSource { impl Source for CombinedSource { fn read(&self, id: &str, ext: &str) -> std::io::Result> { - // we could shortcut on fs if we dont want to check for conflicts + // We could shortcut on fs if we dont check for conflicts let mut entries = self.read_multiple(id, ext); if entries.is_empty() { Err(std::io::ErrorKind::NotFound.into()) @@ -89,7 +86,7 @@ impl Source for CombinedSource { id: &str, f: &mut dyn FnMut(assets_manager::source::DirEntry), ) -> std::io::Result<()> { - // TODO: we should combine the sources + // TODO: We should combine the sources, but this isn't used in veloren self.fs.raw_source().read_dir(id, f) } @@ -98,8 +95,9 @@ impl Source for CombinedSource { || self .plugin_list .read() - .map(|p| p.iter().any(|p| p.cache.raw_source().exists(entry))) - .unwrap_or_default() + .unwrap() + .iter() + .any(|plugin| plugin.cache.raw_source().exists(entry)) } // TODO: Enable hot reloading for plugins @@ -115,29 +113,29 @@ pub struct CombinedCache(AssetCache); impl CombinedCache { pub fn new() -> std::io::Result { - CombinedSource::new().map(|s| Self(AssetCache::with_source(s))) + CombinedSource::new().map(|combined_source| Self(AssetCache::with_source(combined_source))) } - /// combine objects from filesystem and plugins + /// Combine objects from filesystem and plugins pub fn combine( &self, load_from: impl Fn(AnyCache) -> Result, ) -> Result { let mut result = load_from(self.0.raw_source().fs.as_any_cache()); - // report a severe error from the filesystem asset even if later overwritten by + // Report a severe error from the filesystem asset even if later overwritten by // an Ok value from a plugin - if let Err(ref e) = result { - match e + if let Err(ref fs_error) = result { + match fs_error .source() - .and_then(|s| s.downcast_ref::()) - .map(|e| e.kind()) + .and_then(|error_source| error_source.downcast_ref::()) + .map(|io_error| io_error.kind()) { Some(std::io::ErrorKind::NotFound) => (), - _ => tracing::error!("Filesystem asset load {e:?}"), + _ => tracing::error!("Filesystem asset load {fs_error:?}"), } } - for i in self.0.raw_source().plugin_list.read().unwrap().iter() { - match load_from(i.cache.as_any_cache()) { + for plugin in self.0.raw_source().plugin_list.read().unwrap().iter() { + match load_from(plugin.cache.as_any_cache()) { Ok(b) => { result = if let Ok(a) = result { Ok(a.concatenate(b)) @@ -145,15 +143,18 @@ impl CombinedCache { Ok(b) }; }, - // report any error other than NotFound - Err(e) => { - match e + // Report any error other than NotFound + Err(plugin_error) => { + match plugin_error .source() - .and_then(|s| s.downcast_ref::()) - .map(|e| e.kind()) + .and_then(|error_source| error_source.downcast_ref::()) + .map(|io_error| io_error.kind()) { Some(std::io::ErrorKind::NotFound) => (), - _ => tracing::error!("Loading from {:?} failed {e:?}", i.path), + _ => tracing::error!( + "Loading from {:?} failed {plugin_error:?}", + plugin.path + ), } }, } @@ -163,7 +164,6 @@ impl CombinedCache { pub fn register_tar(&self, path: PathBuf) -> std::io::Result<()> { let tar_source = Tar::from_path(&path)?; - //println!("Tar {:?} {:?}", path, tar_source); let cache = AssetCache::with_source(tar_source); self.0 .raw_source() diff --git a/common/assets/src/tar_source.rs b/common/assets/src/tar_source.rs index 8e2c86e914..cf44e342eb 100644 --- a/common/assets/src/tar_source.rs +++ b/common/assets/src/tar_source.rs @@ -10,7 +10,7 @@ use std::{ path::{self, Path, PathBuf}, }; -// derived from the zip source in the assets_manager crate +// Derived from the zip source in the assets_manager crate #[derive(Clone, Hash, PartialEq, Eq)] struct FileDesc(String, String); @@ -82,7 +82,7 @@ fn register_file( match comp { path::Component::Normal(s) => { let segment = s.to_str()?; - // reject paths with extensions + // Reject paths with extensions if segment.contains('.') { return None; } @@ -91,7 +91,7 @@ fn register_file( } parent_id.push_str(segment); }, - // reject paths with non-name components + // Reject paths with non-name components _ => return None, } } @@ -107,11 +107,11 @@ fn register_file( })() .is_none(); if unsupported_path { - tracing::warn!("Unsupported path in tar archive: {path:?}"); + tracing::error!("Unsupported path in tar archive: {path:?}"); } } -// we avoid the extra dependency of sync_file introduced by Zip here by opening +// We avoid the extra dependency of sync_file introduced by Zip here by opening // the file for each read struct Backend(PathBuf); @@ -120,7 +120,7 @@ impl Backend { File::open(self.0.clone()).and_then(|file| { let mut result = vec![0; len]; file.read_at(result.as_mut_slice(), pos) - .map(move |_bytes| result) + .map(move |_num_bytes| result) }) } } @@ -141,12 +141,12 @@ impl Tar { .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; let mut files = HashMap::with_capacity(contents.size_hint().0); let mut dirs = HashMap::new(); - for e in contents.flatten() { - if matches!(e.header().entry_type(), EntryType::Regular) { + for entry in contents.flatten() { + if matches!(entry.header().entry_type(), EntryType::Regular) { register_file( - e.path().map_err(io::Error::other)?.as_ref(), - e.raw_file_position(), - e.size() as usize, + entry.path().map_err(io::Error::other)?.as_ref(), + entry.raw_file_position(), + entry.size() as usize, &mut files, &mut dirs, ); diff --git a/common/state/src/plugin/mod.rs b/common/state/src/plugin/mod.rs index 0db8359221..ae25d31d60 100644 --- a/common/state/src/plugin/mod.rs +++ b/common/state/src/plugin/mod.rs @@ -197,11 +197,11 @@ impl PluginMgr { { info!("Loading plugin at {:?}", entry.path()); Plugin::from_reader(fs::File::open(entry.path()).map_err(PluginError::Io)?).map( - |o| { + |plugin| { if let Err(e) = common::assets::register_tar(entry.path()) { error!("Plugin {:?} tar error {e:?}", entry.path()); } - Some(o) + Some(plugin) }, ) } else { From e22046c5f45fb6f7d97617882806652680b6598f Mon Sep 17 00:00:00 2001 From: Christof Petig Date: Tue, 17 Oct 2023 22:10:19 +0200 Subject: [PATCH 11/11] Address zesterer's comments --- CHANGELOG.md | 2 +- common/assets/Cargo.toml | 3 +- common/assets/src/lib.rs | 2 +- common/assets/src/plugin_cache.rs | 62 +++++++++++------- common/assets/src/tar_source.rs | 2 +- voxygen/Cargo.toml | 2 +- voxygen/src/scene/figure/load.rs | 100 +++++++++++++++--------------- 7 files changed, 95 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a56c9ac58a..763d648f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New arena building in desert cities, suitable for PVP, also NPCs like to watch the fights too - The loading screen now displays status updates for singleplayer server and client initialization progress - New Frost Gigas attacks & AI -- Weapons and armor load from plugins +- Allow plugins to add weapon and armor items ### Changed diff --git a/common/assets/Cargo.toml b/common/assets/Cargo.toml index 1730ad25e4..2ab9f976b0 100644 --- a/common/assets/Cargo.toml +++ b/common/assets/Cargo.toml @@ -25,5 +25,4 @@ walkdir = "2.3.2" [features] hot-reloading = ["assets_manager/hot-reloading"] asset_tweak = ["dep:serde", "hot-reloading"] -hashbrown = ["dep:hashbrown"] -plugins = ["dep:serde", "dep:tar"] +plugins = ["dep:serde", "dep:tar", "dep:hashbrown"] diff --git a/common/assets/src/lib.rs b/common/assets/src/lib.rs index f19ecf36a8..3b9442c83a 100644 --- a/common/assets/src/lib.rs +++ b/common/assets/src/lib.rs @@ -245,7 +245,7 @@ impl Concatenate for Vec { } } -#[cfg(feature = "hashbrown")] +#[cfg(feature = "plugins")] impl Concatenate for hashbrown::HashMap { fn concatenate(mut self, b: Self) -> Self { self.extend(b); diff --git a/common/assets/src/plugin_cache.rs b/common/assets/src/plugin_cache.rs index 261d17e981..0c4ee9eb01 100644 --- a/common/assets/src/plugin_cache.rs +++ b/common/assets/src/plugin_cache.rs @@ -14,8 +14,18 @@ struct PluginEntry { cache: AssetCache, } -/// The source combining filesystem and plugins (typically used via -/// CombinedCache) +/// The location of this asset +enum AssetSource { + FileSystem, + Plugin { index: usize }, +} + +struct SourceAndContents<'a>(AssetSource, FileContent<'a>); + +/// This source combines assets loaded from the filesystem and from plugins. +/// It is typically used via the CombinedCache type. +/// +/// A load will search through all sources and warn about unhandled duplicates. pub struct CombinedSource { fs: AssetCache, plugin_list: RwLock>, @@ -31,34 +41,40 @@ impl CombinedSource { } impl CombinedSource { - fn read_multiple(&self, id: &str, ext: &str) -> Vec<(Option, FileContent<'_>)> { + /// Look for an asset in all known sources + fn read_multiple(&self, id: &str, ext: &str) -> Vec> { let mut result = Vec::new(); if let Ok(file_entry) = self.fs.raw_source().read(id, ext) { - result.push((None, file_entry)); + result.push(SourceAndContents(AssetSource::FileSystem, file_entry)); } for (n, p) in self.plugin_list.read().unwrap().iter().enumerate() { if let Ok(entry) = p.cache.raw_source().read(id, ext) { // the data is behind an RwLockReadGuard, so own it for returning - result.push((Some(n), match entry { - FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), - FileContent::Buffer(b) => FileContent::Buffer(b), - FileContent::Owned(s) => FileContent::Buffer(Vec::from(s.as_ref().as_ref())), - })); + result.push(SourceAndContents( + AssetSource::Plugin { index: n }, + match entry { + FileContent::Slice(s) => FileContent::Buffer(Vec::from(s)), + FileContent::Buffer(b) => FileContent::Buffer(b), + FileContent::Owned(s) => { + FileContent::Buffer(Vec::from(s.as_ref().as_ref())) + }, + }, + )); } } result } - // We don't want to keep the lock, so we clone - fn plugin_path(&self, index: Option) -> Option { - if let Some(index) = index { - self.plugin_list + /// Return the path of a source + fn plugin_path(&self, index: &AssetSource) -> Option { + match index { + AssetSource::FileSystem => Some(ASSETS_PATH.clone()), + AssetSource::Plugin { index } => self.plugin_list .read() .unwrap() - .get(index) - .map(|plugin| plugin.path.clone()) - } else { - None + .get(*index) + // We don't want to keep the lock, so we clone + .map(|plugin| plugin.path.clone()), } } } @@ -71,12 +87,11 @@ impl Source for CombinedSource { Err(std::io::ErrorKind::NotFound.into()) } else { if entries.len() > 1 { - let plugina = self.plugin_path(entries[0].0); - let pluginb = self.plugin_path(entries[1].0); - let patha = plugina.as_ref().unwrap_or(&ASSETS_PATH); - let pathb = pluginb.as_ref().unwrap_or(&ASSETS_PATH); + let patha = self.plugin_path(&entries[0].0); + let pathb = self.plugin_path(&entries[1].0); tracing::error!("Duplicate asset {id} in {patha:?} and {pathb:?}"); } + // unconditionally return the first asset found Ok(entries.swap_remove(0).1) } } @@ -119,7 +134,7 @@ impl CombinedCache { /// Combine objects from filesystem and plugins pub fn combine( &self, - load_from: impl Fn(AnyCache) -> Result, + mut load_from: impl FnMut(AnyCache) -> Result, ) -> Result { let mut result = load_from(self.0.raw_source().fs.as_any_cache()); // Report a severe error from the filesystem asset even if later overwritten by @@ -162,6 +177,8 @@ impl CombinedCache { result } + /// Add a tar archive (a plugin) to the system. + /// All files in that tar file become potential assets. pub fn register_tar(&self, path: PathBuf) -> std::io::Result<()> { let tar_source = Tar::from_path(&path)?; let cache = AssetCache::with_source(tar_source); @@ -175,6 +192,7 @@ impl CombinedCache { } } +// Delegate all cache operations directly to the contained cache object impl std::ops::Deref for CombinedCache { type Target = AssetCache; diff --git a/common/assets/src/tar_source.rs b/common/assets/src/tar_source.rs index cf44e342eb..2cf0e313f7 100644 --- a/common/assets/src/tar_source.rs +++ b/common/assets/src/tar_source.rs @@ -119,7 +119,7 @@ impl Backend { fn read(&self, pos: u64, len: usize) -> std::io::Result> { File::open(self.0.clone()).and_then(|file| { let mut result = vec![0; len]; - file.read_at(result.as_mut_slice(), pos) + file.read_exact_at(result.as_mut_slice(), pos) .map(move |_num_bytes| result) }) } diff --git a/voxygen/Cargo.toml b/voxygen/Cargo.toml index ef7c7a4bde..883d1d7adf 100644 --- a/voxygen/Cargo.toml +++ b/voxygen/Cargo.toml @@ -27,7 +27,7 @@ singleplayer = ["server"] simd = ["vek/platform_intrinsics"] tracy = ["common-frontend/tracy", "client/tracy"] tracy-memory = ["tracy"] # enables heap profiling with tracy -plugins = ["client/plugins", "common-assets/plugins", "common-assets/hashbrown"] +plugins = ["client/plugins", "common-assets/plugins"] egui-ui = ["voxygen-egui", "egui", "egui_wgpu_backend", "egui_winit_platform"] shaderc-from-source = ["shaderc/build-from-source"] discord = ["discord-sdk"] diff --git a/voxygen/src/scene/figure/load.rs b/voxygen/src/scene/figure/load.rs index f17d166515..7ff430d1b2 100644 --- a/voxygen/src/scene/figure/load.rs +++ b/voxygen/src/scene/figure/load.rs @@ -178,7 +178,7 @@ macro_rules! make_vox_spec { } } } -macro_rules! concatenate_tuple { +macro_rules! impl_concatenate_for_wrapper { ($name:ty) => { impl Concatenate for $name { fn concatenate(self, b: Self) -> Self { Self(self.0.concatenate(b.0)) } @@ -375,7 +375,7 @@ impl HumHeadSpec { ) } } -concatenate_tuple!(HumHeadSpec); +impl_concatenate_for_wrapper!(HumHeadSpec); // Armor aspects should be in the same order, top to bottom. // These seem overly split up, but wanted to keep the armor seperated @@ -398,43 +398,43 @@ impl Concatenate for ArmorVoxSpecMap { } #[derive(Deserialize)] struct HumArmorShoulderSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorShoulderSpec); +impl_concatenate_for_wrapper!(HumArmorShoulderSpec); #[derive(Deserialize)] struct HumArmorChestSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorChestSpec); +impl_concatenate_for_wrapper!(HumArmorChestSpec); #[derive(Deserialize)] struct HumArmorHandSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorHandSpec); +impl_concatenate_for_wrapper!(HumArmorHandSpec); #[derive(Deserialize)] struct HumArmorBeltSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorBeltSpec); +impl_concatenate_for_wrapper!(HumArmorBeltSpec); #[derive(Deserialize)] struct HumArmorBackSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorBackSpec); +impl_concatenate_for_wrapper!(HumArmorBackSpec); #[derive(Deserialize)] struct HumArmorPantsSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorPantsSpec); +impl_concatenate_for_wrapper!(HumArmorPantsSpec); #[derive(Deserialize)] struct HumArmorFootSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorFootSpec); +impl_concatenate_for_wrapper!(HumArmorFootSpec); #[derive(Deserialize)] struct HumMainWeaponSpec(HashMap); -concatenate_tuple!(HumMainWeaponSpec); +impl_concatenate_for_wrapper!(HumMainWeaponSpec); #[derive(Deserialize)] struct HumModularComponentSpec(HashMap); -concatenate_tuple!(HumModularComponentSpec); +impl_concatenate_for_wrapper!(HumModularComponentSpec); #[derive(Deserialize)] struct HumArmorLanternSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorLanternSpec); +impl_concatenate_for_wrapper!(HumArmorLanternSpec); #[derive(Deserialize)] struct HumArmorGliderSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorGliderSpec); +impl_concatenate_for_wrapper!(HumArmorGliderSpec); #[derive(Deserialize)] struct HumArmorHeadSpec(ArmorVoxSpecMap<(Species, BodyType, String), ArmorVoxSpec>); -concatenate_tuple!(HumArmorHeadSpec); +impl_concatenate_for_wrapper!(HumArmorHeadSpec); #[derive(Deserialize)] struct HumArmorTabardSpec(ArmorVoxSpecMap); -concatenate_tuple!(HumArmorTabardSpec); +impl_concatenate_for_wrapper!(HumArmorTabardSpec); make_vox_spec!( Body, @@ -1105,7 +1105,7 @@ fn mesh_hold() -> BoneMeshes { ////// #[derive(Deserialize)] struct QuadrupedSmallCentralSpec(HashMap<(QSSpecies, QSBodyType), SidedQSCentralVoxSpec>); -concatenate_tuple!(QuadrupedSmallCentralSpec); +impl_concatenate_for_wrapper!(QuadrupedSmallCentralSpec); #[derive(Deserialize)] struct SidedQSCentralVoxSpec { @@ -1123,7 +1123,7 @@ struct QuadrupedSmallCentralSubSpec { #[derive(Deserialize)] struct QuadrupedSmallLateralSpec(HashMap<(QSSpecies, QSBodyType), SidedQSLateralVoxSpec>); -concatenate_tuple!(QuadrupedSmallLateralSpec); +impl_concatenate_for_wrapper!(QuadrupedSmallLateralSpec); #[derive(Deserialize)] struct SidedQSLateralVoxSpec { @@ -1322,7 +1322,7 @@ impl QuadrupedSmallLateralSpec { ////// #[derive(Deserialize)] struct QuadrupedMediumCentralSpec(HashMap<(QMSpecies, QMBodyType), SidedQMCentralVoxSpec>); -concatenate_tuple!(QuadrupedMediumCentralSpec); +impl_concatenate_for_wrapper!(QuadrupedMediumCentralSpec); #[derive(Deserialize)] struct SidedQMCentralVoxSpec { @@ -1344,7 +1344,7 @@ struct QuadrupedMediumCentralSubSpec { #[derive(Deserialize)] struct QuadrupedMediumLateralSpec(HashMap<(QMSpecies, QMBodyType), SidedQMLateralVoxSpec>); -concatenate_tuple!(QuadrupedMediumLateralSpec); +impl_concatenate_for_wrapper!(QuadrupedMediumLateralSpec); #[derive(Deserialize)] struct SidedQMLateralVoxSpec { leg_fl: QuadrupedMediumLateralSubSpec, @@ -1700,7 +1700,7 @@ impl QuadrupedMediumLateralSpec { ////// #[derive(Deserialize)] struct BirdMediumCentralSpec(HashMap<(BMSpecies, BMBodyType), SidedBMCentralVoxSpec>); -concatenate_tuple!(BirdMediumCentralSpec); +impl_concatenate_for_wrapper!(BirdMediumCentralSpec); #[derive(Deserialize)] struct SidedBMCentralVoxSpec { @@ -1718,7 +1718,7 @@ struct BirdMediumCentralSubSpec { #[derive(Deserialize)] struct BirdMediumLateralSpec(HashMap<(BMSpecies, BMBodyType), SidedBMLateralVoxSpec>); -concatenate_tuple!(BirdMediumLateralSpec); +impl_concatenate_for_wrapper!(BirdMediumLateralSpec); #[derive(Deserialize)] struct SidedBMLateralVoxSpec { @@ -1956,7 +1956,7 @@ impl BirdMediumLateralSpec { ////// #[derive(Deserialize)] struct TheropodCentralSpec(HashMap<(TSpecies, TBodyType), SidedTCentralVoxSpec>); -concatenate_tuple!(TheropodCentralSpec); +impl_concatenate_for_wrapper!(TheropodCentralSpec); #[derive(Deserialize)] struct SidedTCentralVoxSpec { @@ -1977,7 +1977,7 @@ struct TheropodCentralSubSpec { } #[derive(Deserialize)] struct TheropodLateralSpec(HashMap<(TSpecies, TBodyType), SidedTLateralVoxSpec>); -concatenate_tuple!(TheropodLateralSpec); +impl_concatenate_for_wrapper!(TheropodLateralSpec); #[derive(Deserialize)] struct SidedTLateralVoxSpec { @@ -2288,7 +2288,7 @@ impl TheropodLateralSpec { ////// #[derive(Deserialize)] struct ArthropodCentralSpec(HashMap<(ASpecies, ABodyType), SidedACentralVoxSpec>); -concatenate_tuple!(ArthropodCentralSpec); +impl_concatenate_for_wrapper!(ArthropodCentralSpec); #[derive(Deserialize)] struct SidedACentralVoxSpec { @@ -2304,7 +2304,7 @@ struct ArthropodCentralSubSpec { } #[derive(Deserialize)] struct ArthropodLateralSpec(HashMap<(ASpecies, ABodyType), SidedALateralVoxSpec>); -concatenate_tuple!(ArthropodLateralSpec); +impl_concatenate_for_wrapper!(ArthropodLateralSpec); #[derive(Deserialize)] struct SidedALateralVoxSpec { @@ -2690,7 +2690,7 @@ impl ArthropodLateralSpec { ////// #[derive(Deserialize)] struct FishMediumCentralSpec(HashMap<(FMSpecies, FMBodyType), SidedFMCentralVoxSpec>); -concatenate_tuple!(FishMediumCentralSpec); +impl_concatenate_for_wrapper!(FishMediumCentralSpec); #[derive(Deserialize)] struct SidedFMCentralVoxSpec { @@ -2709,7 +2709,7 @@ struct FishMediumCentralSubSpec { } #[derive(Deserialize)] struct FishMediumLateralSpec(HashMap<(FMSpecies, FMBodyType), SidedFMLateralVoxSpec>); -concatenate_tuple!(FishMediumLateralSpec); +impl_concatenate_for_wrapper!(FishMediumLateralSpec); #[derive(Deserialize)] struct SidedFMLateralVoxSpec { fin_l: FishMediumLateralSubSpec, @@ -2898,7 +2898,7 @@ impl FishMediumLateralSpec { ////// #[derive(Deserialize)] struct FishSmallCentralSpec(HashMap<(FSSpecies, FSBodyType), SidedFSCentralVoxSpec>); -concatenate_tuple!(FishSmallCentralSpec); +impl_concatenate_for_wrapper!(FishSmallCentralSpec); #[derive(Deserialize)] struct SidedFSCentralVoxSpec { @@ -2914,7 +2914,7 @@ struct FishSmallCentralSubSpec { } #[derive(Deserialize)] struct FishSmallLateralSpec(HashMap<(FSSpecies, FSBodyType), SidedFSLateralVoxSpec>); -concatenate_tuple!(FishSmallLateralSpec); +impl_concatenate_for_wrapper!(FishSmallLateralSpec); #[derive(Deserialize)] struct SidedFSLateralVoxSpec { fin_l: FishSmallLateralSubSpec, @@ -3045,25 +3045,25 @@ impl FishSmallLateralSpec { #[derive(Deserialize)] struct BipedSmallWeaponSpec(HashMap); -concatenate_tuple!(BipedSmallWeaponSpec); +impl_concatenate_for_wrapper!(BipedSmallWeaponSpec); #[derive(Deserialize)] struct BipedSmallArmorHeadSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorHeadSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorHeadSpec); #[derive(Deserialize)] struct BipedSmallArmorHandSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorHandSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorHandSpec); #[derive(Deserialize)] struct BipedSmallArmorFootSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorFootSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorFootSpec); #[derive(Deserialize)] struct BipedSmallArmorChestSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorChestSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorChestSpec); #[derive(Deserialize)] struct BipedSmallArmorPantsSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorPantsSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorPantsSpec); #[derive(Deserialize)] struct BipedSmallArmorTailSpec(ArmorVoxSpecMap); -concatenate_tuple!(BipedSmallArmorTailSpec); +impl_concatenate_for_wrapper!(BipedSmallArmorTailSpec); make_vox_spec!( biped_small::Body, struct BipedSmallSpec { @@ -3327,7 +3327,7 @@ impl BipedSmallWeaponSpec { ////// #[derive(Deserialize)] struct DragonCentralSpec(HashMap<(DSpecies, DBodyType), SidedDCentralVoxSpec>); -concatenate_tuple!(DragonCentralSpec); +impl_concatenate_for_wrapper!(DragonCentralSpec); #[derive(Deserialize)] struct SidedDCentralVoxSpec { @@ -3349,7 +3349,7 @@ struct DragonCentralSubSpec { #[derive(Deserialize)] struct DragonLateralSpec(HashMap<(DSpecies, DBodyType), SidedDLateralVoxSpec>); -concatenate_tuple!(DragonLateralSpec); +impl_concatenate_for_wrapper!(DragonLateralSpec); #[derive(Deserialize)] struct SidedDLateralVoxSpec { @@ -3700,7 +3700,7 @@ impl DragonLateralSpec { ////// #[derive(Deserialize)] struct BirdLargeCentralSpec(HashMap<(BLASpecies, BLABodyType), SidedBLACentralVoxSpec>); -concatenate_tuple!(BirdLargeCentralSpec); +impl_concatenate_for_wrapper!(BirdLargeCentralSpec); #[derive(Deserialize)] struct SidedBLACentralVoxSpec { @@ -3721,7 +3721,7 @@ struct BirdLargeCentralSubSpec { #[derive(Deserialize)] struct BirdLargeLateralSpec(HashMap<(BLASpecies, BLABodyType), SidedBLALateralVoxSpec>); -concatenate_tuple!(BirdLargeLateralSpec); +impl_concatenate_for_wrapper!(BirdLargeLateralSpec); #[derive(Deserialize)] struct SidedBLALateralVoxSpec { @@ -4105,7 +4105,7 @@ impl BirdLargeLateralSpec { ////// #[derive(Deserialize)] struct BipedLargeCentralSpec(HashMap<(BLSpecies, BLBodyType), SidedBLCentralVoxSpec>); -concatenate_tuple!(BipedLargeCentralSpec); +impl_concatenate_for_wrapper!(BipedLargeCentralSpec); #[derive(Deserialize)] struct SidedBLCentralVoxSpec { @@ -4125,7 +4125,7 @@ struct BipedLargeCentralSubSpec { #[derive(Deserialize)] struct BipedLargeLateralSpec(HashMap<(BLSpecies, BLBodyType), SidedBLLateralVoxSpec>); -concatenate_tuple!(BipedLargeLateralSpec); +impl_concatenate_for_wrapper!(BipedLargeLateralSpec); #[derive(Deserialize)] struct SidedBLLateralVoxSpec { @@ -4147,10 +4147,10 @@ struct BipedLargeLateralSubSpec { } #[derive(Deserialize)] struct BipedLargeMainSpec(HashMap); -concatenate_tuple!(BipedLargeMainSpec); +impl_concatenate_for_wrapper!(BipedLargeMainSpec); #[derive(Deserialize)] struct BipedLargeSecondSpec(HashMap); -concatenate_tuple!(BipedLargeSecondSpec); +impl_concatenate_for_wrapper!(BipedLargeSecondSpec); make_vox_spec!( biped_large::Body, struct BipedLargeSpec { @@ -4528,7 +4528,7 @@ impl BipedLargeSecondSpec { ////// #[derive(Deserialize)] struct GolemCentralSpec(HashMap<(GSpecies, GBodyType), SidedGCentralVoxSpec>); -concatenate_tuple!(GolemCentralSpec); +impl_concatenate_for_wrapper!(GolemCentralSpec); #[derive(Deserialize)] struct SidedGCentralVoxSpec { @@ -4547,7 +4547,7 @@ struct GolemCentralSubSpec { #[derive(Deserialize)] struct GolemLateralSpec(HashMap<(GSpecies, GBodyType), SidedGLateralVoxSpec>); -concatenate_tuple!(GolemLateralSpec); +impl_concatenate_for_wrapper!(GolemLateralSpec); #[derive(Deserialize)] struct SidedGLateralVoxSpec { @@ -4840,7 +4840,7 @@ impl GolemLateralSpec { ////// #[derive(Deserialize)] struct QuadrupedLowCentralSpec(HashMap<(QLSpecies, QLBodyType), SidedQLCentralVoxSpec>); -concatenate_tuple!(QuadrupedLowCentralSpec); +impl_concatenate_for_wrapper!(QuadrupedLowCentralSpec); #[derive(Deserialize)] struct SidedQLCentralVoxSpec { @@ -4861,7 +4861,7 @@ struct QuadrupedLowCentralSubSpec { #[derive(Deserialize)] struct QuadrupedLowLateralSpec(HashMap<(QLSpecies, QLBodyType), SidedQLLateralVoxSpec>); -concatenate_tuple!(QuadrupedLowLateralSpec); +impl_concatenate_for_wrapper!(QuadrupedLowLateralSpec); #[derive(Deserialize)] struct SidedQLLateralVoxSpec { front_left: QuadrupedLowLateralSubSpec, @@ -5191,7 +5191,7 @@ impl ObjectCentralSpec { (central, Vec3::from(spec.bone1.offset)) } } -concatenate_tuple!(ObjectCentralSpec); +impl_concatenate_for_wrapper!(ObjectCentralSpec); struct ModelWithOptionalIndex(String, u32); @@ -5229,7 +5229,7 @@ impl<'de> Deserialize<'de> for ModelWithOptionalIndex { #[derive(Deserialize)] struct ItemDropCentralSpec(HashMap); -concatenate_tuple!(ItemDropCentralSpec); +impl_concatenate_for_wrapper!(ItemDropCentralSpec); make_vox_spec!( item_drop::Body,