Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'alphyr/assets_manager_v0.11' into 'master'

Browse Source 
Update `assets_manager` to v0.11

See merge request veloren/veloren!4248
This commit is contained in:
Illia Denysenko 2024-01-20 15:46:16 +00:00
commit dba8c94030
16 changed files with 79 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Cargo.lock generated
View File

@ -316,9 +316,9 @@ dependencies = [
[[package]]
name = "assets_manager"
version = "0.10.2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f00425f4c1f3349b35daf0a73477249f6574fe89f4b9d76aca0b2a1356886b3b"
checksum = "3725d3fcae7767855256e1f627df67ac8ca675c55fde1bf784d5669d1ddc7261"
dependencies = [
"ab_glyph",
"ahash 0.8.7",
@ -326,10 +326,11 @@ dependencies = [
"crossbeam-channel",
"log",
"notify",
"parking_lot 0.12.1",
"ron 0.8.1",
"serde",
"serde_json",
"sync_file",
"tar",
]
[[package]]
@ -6208,6 +6209,15 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_file"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a49b72df45d9c2d5fbb13b02c97437a3175d8c9860297297597d3ed715e0f046"
dependencies = [
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
@ -7008,7 +7018,6 @@ dependencies = [
"lazy_static",
"ron 0.8.1",
"serde",
"tar",
"tracing",
"walkdir",
"wavefront",

13
client/i18n/src/lib.rs
View File

@ -16,7 +16,9 @@ use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, io};
use assets::{source::DirEntry, AssetExt, AssetGuard, AssetHandle, ReloadWatcher, SharedString};
use assets::{
source::DirEntry, AssetExt, AssetHandle, AssetReadGuard, ReloadWatcher, SharedString,
};
use common_assets as assets;
use common_i18n::{Content, LocalizationArg};
use tracing::warn;
@ -167,7 +169,7 @@ impl assets::Compound for Language {
let mut bundle = FluentBundle::new_concurrent(vec![lang_id]);
// Here go dragons
for id in cache.load_dir::<raw::Resource>(path, true)?.ids() {
for id in cache.load_rec_dir::<raw::Resource>(path)?.read().ids() {
match cache.load(id) {
Ok(handle) => {
let source: &raw::Resource = &handle.read();
@ -229,8 +231,8 @@ pub type Localization = LocalizationGuard;
/// RAII guard returned from [`LocalizationHandle::read()`], resembles
/// [`AssetGuard`]
pub struct LocalizationGuard {
active: AssetGuard<Language>,
fallback: Option<AssetGuard<Language>>,
active: AssetReadGuard<Language>,
fallback: Option<AssetReadGuard<Language>>,
}
impl LocalizationGuard {
@ -592,8 +594,9 @@ struct LocalizationList(Vec<LanguageMetadata>);
impl assets::Compound for LocalizationList {
fn load(cache: assets::AnyCache, specifier: &SharedString) -> Result<Self, assets::BoxedError> {
// List language directories
let languages = assets::load_dir::<FindManifests>(specifier, false)
let languages = assets::load_rec_dir::<FindManifests>(specifier)
.unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e))
.read()
.ids()
.filter_map(|spec| cache.load::<raw::Manifest>(spec).ok())
.map(|localization| localization.read().metadata.clone())

5
common/assets/Cargo.toml
View File

@ -7,13 +7,12 @@ version = "0.10.0"
[dependencies]
lazy_static = { workspace = true }
assets_manager = {version = "0.10", features = ["bincode", "ron", "json"]}
assets_manager = { version = "0.11.1", features = ["bincode", "ron", "json"] }
ron = { workspace = true }
dot_vox = "5.1"
wavefront = "0.2" # TODO: Use vertex-colors branch when we have models that have them
image = { workspace = true }
tracing = { workspace = true }
tar = { version = "0.4.37", optional = true }
hashbrown = { workspace = true }
# asset tweak
@ -25,4 +24,4 @@ walkdir = "2.3.2"
[features]
hot-reloading = ["assets_manager/hot-reloading"]
asset_tweak = ["dep:serde", "hot-reloading"]
plugins = ["dep:serde", "dep:tar"]
plugins = ["dep:serde", "assets_manager/tar"]

7
common/assets/src/fs.rs
View File

@ -1,7 +1,7 @@
use std::{fs, io};
use assets_manager::{
hot_reloading::{DynUpdateSender, EventSender, FsWatcherBuilder},
hot_reloading::{EventSender, FsWatcherBuilder},
source::{DirEntry, FileContent, FileSystem as RawFs, Source},
BoxedError,
};
@ -94,7 +94,7 @@ impl Source for FileSystem {
fn make_source(&self) -> Option<Box<dyn Source + Send>> { Some(Box::new(self.clone())) }
fn configure_hot_reloading(&self, events: EventSender) -> Result<DynUpdateSender, BoxedError> {
fn configure_hot_reloading(&self, events: EventSender) -> Result<(), BoxedError> {
let mut builder = FsWatcherBuilder::new()?;
if let Some(dir) = &self.override_dir {
@ -102,6 +102,7 @@ impl Source for FileSystem {
}
builder.watch(self.default.root().to_owned())?;
Ok(builder.build(events))
builder.build(events);
Ok(())
}
}

47
common/assets/src/lib.rs
View File

@ -25,7 +25,6 @@ pub use assets_manager::{
mod fs;
#[cfg(feature = "plugins")] mod plugin_cache;
#[cfg(feature = "plugins")] mod tar_source;
mod walk;
pub use walk::{walk_tree, Walk};
@ -48,9 +47,9 @@ pub fn start_hot_reloading() { ASSETS.enhance_hot_reloading(); }
#[cfg(feature = "plugins")]
pub fn register_tar(path: PathBuf) -> std::io::Result<()> { ASSETS.register_tar(path) }
pub type AssetHandle<T> = assets_manager::Handle<'static, T>;
pub type AssetGuard<T> = assets_manager::AssetGuard<'static, T>;
pub type AssetDirHandle<T> = assets_manager::DirHandle<'static, T>;
pub type AssetHandle<T> = &'static assets_manager::Handle<T>;
pub type AssetReadGuard<T> = assets_manager::AssetReadGuard<'static, T>;
pub type AssetDirHandle<T> = AssetHandle<assets_manager::RecursiveDirectory<T>>;
pub type ReloadWatcher = assets_manager::ReloadWatcher<'static>;
/// The Asset trait, which is implemented by all structures that have their data
@ -71,7 +70,7 @@ pub trait AssetExt: Sized + Send + Sync + 'static {
where
Self: Clone,
{
Self::load(specifier).map(AssetHandle::cloned)
Self::load(specifier).map(|h| h.cloned())
}
fn load_or_insert_with(
@ -143,7 +142,7 @@ pub trait CacheCombined<'a> {
fn load_and_combine<A: Compound + Concatenate>(
self,
id: &str,
) -> Result<assets_manager::Handle<'a, A>, BoxedError>;
) -> Result<&'a assets_manager::Handle<A>, BoxedError>;
}
/// Loads directory and all files in it
@ -154,39 +153,9 @@ pub trait CacheCombined<'a> {
///
/// When loading a directory recursively, directories that can't be read are
/// ignored.
pub fn load_dir<T: DirLoadable>(
specifier: &str,
recursive: bool,
) -> Result<AssetDirHandle<T>, Error> {
pub fn load_rec_dir<T: DirLoadable>(specifier: &str) -> Result<AssetDirHandle<T>, Error> {
let specifier = specifier.strip_suffix(".*").unwrap_or(specifier);
ASSETS.load_dir(specifier, recursive)
}
/// Loads directory and all files in it
///
/// # Panics
/// 1) If can't load directory (filesystem errors)
/// 2) If file can't be loaded (parsing problem)
#[track_caller]
pub fn read_expect_dir<T: DirLoadable + Compound>(
specifier: &str,
recursive: bool,
) -> impl Iterator<Item = AssetGuard<T>> {
#[track_caller]
#[cold]
fn expect_failed(err: Error) -> ! {
panic!(
"Failed loading directory: {} (error={:?})",
err.id(),
err.reason()
)
}
// Avoid using `unwrap_or_else` to avoid breaking `#[track_caller]`
match load_dir::<T>(specifier, recursive) {
Ok(dir) => dir.ids().map(|entry| T::load_expect(entry).read()),
Err(err) => expect_failed(err),
}
ASSETS.load_rec_dir(specifier)
}
impl<T: Compound> AssetExt for T {
@ -203,7 +172,7 @@ impl<'a> CacheCombined<'a> for AnyCache<'a> {
fn load_and_combine<A: Compound + Concatenate>(
self,
specifier: &str,
) -> Result<assets_manager::Handle<'a, A>, BoxedError> {
) -> Result<&'a assets_manager::Handle<A>, BoxedError> {
#[cfg(feature = "plugins")]
{
self.get_cached(specifier).map_or_else(

10
common/assets/src/plugin_cache.rs
View File

@ -2,10 +2,10 @@ use std::{path::PathBuf, sync::RwLock};
use crate::Concatenate;
use super::{fs::FileSystem, tar_source::Tar, ASSETS_PATH};
use super::{fs::FileSystem, ASSETS_PATH};
use assets_manager::{
hot_reloading::{DynUpdateSender, EventSender},
source::{FileContent, Source},
hot_reloading::EventSender,
source::{FileContent, Source, Tar},
AnyCache, AssetCache, BoxedError,
};
@ -118,7 +118,7 @@ impl Source for CombinedSource {
// TODO: Enable hot reloading for plugins
fn make_source(&self) -> Option<Box<dyn Source + Send>> { self.fs.raw_source().make_source() }
fn configure_hot_reloading(&self, events: EventSender) -> Result<DynUpdateSender, BoxedError> {
fn configure_hot_reloading(&self, events: EventSender) -> Result<(), BoxedError> {
self.fs.raw_source().configure_hot_reloading(events)
}
}
@ -180,7 +180,7 @@ impl CombinedCache {
/// Add a tar archive (a plugin) to the system.
/// All files in that tar file become potential assets.
pub fn register_tar(&self, path: PathBuf) -> std::io::Result<()> {
let tar_source = Tar::from_path(&path)?;
let tar_source = Tar::open(&path)?;
let cache = AssetCache::with_source(tar_source);
self.0
.raw_source()

159
common/assets/src/tar_source.rs
View File

@ -1,159 +0,0 @@
use assets_manager::source::{DirEntry, FileContent, Source};
use hashbrown::HashMap;
use tar::EntryType;
use std::{
fmt,
fs::File,
io::{self, Read, Seek, SeekFrom},
path::{self, Path, PathBuf},
};
// Derived from the zip source in the assets_manager crate
#[derive(Clone, Hash, PartialEq, Eq)]
struct FileDesc(String, String);
impl hashbrown::Equivalent<FileDesc> for (&str, &str) {
fn equivalent(&self, key: &FileDesc) -> bool { self.0 == key.0 && self.1 == key.1 }
}
impl fmt::Debug for FileDesc {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("FileDesc")
.field("id", &self.0)
.field("ext", &self.1)
.finish()
}
}
impl FileDesc {
fn as_dir_entry(&self) -> DirEntry { DirEntry::File(&self.0, &self.1) }
}
/// Register a file of an archive in maps, components in asset ids are separated
/// by points
fn register_file(
path: &Path,
position: u64,
length: usize,
files: &mut HashMap<FileDesc, (u64, usize)>,
dirs: &mut HashMap<String, Vec<FileDesc>>,
) {
// Parse the path and register it.
let mut parent_id = String::default();
// The closure is used as a cheap `try` block.
let unsupported_path = (|| {
let parent = path.parent()?;
for comp in parent.components() {
match comp {
path::Component::Normal(s) => {
let segment = s.to_str()?;
// Reject paths with extensions
if segment.contains('.') {
return None;
}
if !parent_id.is_empty() {
parent_id.push('.');
}
parent_id.push_str(segment);
},
// Reject paths with non-name components
_ => return None,
}
}
let file_id = parent_id.clone() + "." + path.file_stem()?.to_str()?;
// Register the file in the maps.
let ext = path.extension().unwrap_or_default().to_str()?.to_owned();
let desc = FileDesc(file_id, ext);
files.insert(desc.clone(), (position, length));
dirs.entry(parent_id).or_default().push(desc);
Some(())
})()
.is_none();
if unsupported_path {
tracing::error!("Unsupported path in tar archive: {path:?}");
}
}
// We avoid the extra dependency of sync_file introduced by Zip here by opening
// the file for each read
struct Backend(PathBuf);
impl Backend {
fn read(&self, pos: u64, len: usize) -> std::io::Result<Vec<u8>> {
File::open(self.0.clone()).and_then(|mut file| {
file.seek(SeekFrom::Start(pos)).and_then(|_| {
let mut result = vec![0; len];
file.read_exact(result.as_mut_slice())
.map(move |_num_bytes| result)
})
})
}
}
pub struct Tar {
files: HashMap<FileDesc, (u64, usize)>,
dirs: HashMap<String, Vec<FileDesc>>,
backend: Backend,
}
impl Tar {
/// Creates a `Tar` from a file
pub fn from_path(path: &Path) -> io::Result<Tar> {
let file = File::open(path)?;
let mut tar = tar::Archive::new(file);
let contents = tar
.entries()
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let mut files = HashMap::with_capacity(contents.size_hint().0);
let mut dirs = HashMap::new();
for entry in contents.flatten() {
if matches!(entry.header().entry_type(), EntryType::Regular) {
register_file(
entry.path().map_err(io::Error::other)?.as_ref(),
entry.raw_file_position(),
entry.size() as usize,
&mut files,
&mut dirs,
);
}
}
Ok(Tar {
files,
dirs,
backend: Backend(path.to_path_buf()),
})
}
}
impl Source for Tar {
fn read(&self, id: &str, ext: &str) -> io::Result<FileContent> {
let id = *self.files.get(&(id, ext)).ok_or(io::ErrorKind::NotFound)?;
self.backend.read(id.0, id.1).map(FileContent::Buffer)
}
fn read_dir(&self, id: &str, f: &mut dyn FnMut(DirEntry)) -> io::Result<()> {
let dir = self.dirs.get(id).ok_or(io::ErrorKind::NotFound)?;
dir.iter().map(FileDesc::as_dir_entry).for_each(f);
Ok(())
}
fn exists(&self, entry: DirEntry) -> bool {
match entry {
DirEntry::File(id, ext) => self.files.contains_key(&(id, ext)),
DirEntry::Directory(id) => self.dirs.contains_key(id),
}
}
}
impl fmt::Debug for Tar {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Tar")
.field("files", &self.files)
.field("dirs", &self.dirs)
.finish()
}
}

6
common/src/bin/csv_export/main.rs
View File

@ -489,10 +489,10 @@ fn entity_drops(entity_config: &str) -> Result<(), Box<dyn Error>> {
}
if entity_config.eq_ignore_ascii_case("all") {
let configs = assets::load_dir::<EntityConfig>("common.entity", true)
let configs = assets::load_rec_dir::<EntityConfig>("common.entity")
.expect("Entity files moved somewhere else maybe?")
.ids();
for config in configs {
.read();
for config in configs.ids() {
write_entity_loot(&mut wtr, config)?;
}
} else {

14
common/src/comp/inventory/item/mod.rs
View File

@ -975,8 +975,11 @@ impl Item {
/// asset glob pattern
pub fn new_from_asset_glob(asset_glob: &str) -> Result<Vec<Self>, Error> {
let specifier = asset_glob.strip_suffix(".*").unwrap_or(asset_glob);
let defs = assets::load_dir::<RawItemDef>(specifier, true)?;
defs.ids().map(|id| Item::new_from_asset(id)).collect()
let defs = assets::load_rec_dir::<RawItemDef>(specifier)?;
defs.read()
.ids()
.map(|id| Item::new_from_asset(id))
.collect()
}
/// Creates a new instance of an `Item from the provided asset identifier if
@ -1632,18 +1635,19 @@ pub fn all_item_defs_expect() -> Vec<String> {
/// Returns all item asset specifiers
pub fn try_all_item_defs() -> Result<Vec<String>, Error> {
let defs = assets::load_dir::<RawItemDef>("common.items", true)?;
Ok(defs.ids().map(|id| id.to_string()).collect())
let defs = assets::load_rec_dir::<RawItemDef>("common.items")?;
Ok(defs.read().ids().map(|id| id.to_string()).collect())
}
/// Designed to return all possible items, including modulars.
/// And some impossible too, like ItemKind::TagExamples.
pub fn all_items_expect() -> Vec<Item> {
let defs = assets::load_dir::<RawItemDef>("common.items", true)
let defs = assets::load_rec_dir::<RawItemDef>("common.items")
.expect("failed to load item asset directory");
// Grab all items from assets
let mut asset_items: Vec<Item> = defs
.read()
.ids()
.map(|id| Item::new_from_asset_expect(id))
.collect();

8
common/src/comp/inventory/loadout_builder.rs
View File

@ -1437,9 +1437,9 @@ mod tests {
// One for asset itself and second if it serves as a base for other asset.
#[test]
fn validate_all_loadout_assets() {
let loadouts = assets::load_dir::<LoadoutSpec>("common.loadout", true)
let loadouts = assets::load_rec_dir::<LoadoutSpec>("common.loadout")
.expect("failed to load loadout directory");
for loadout_id in loadouts.ids() {
for loadout_id in loadouts.read().ids() {
let loadout =
LoadoutSpec::load_cloned(loadout_id).expect("failed to load loadout asset");
loadout
@ -1451,10 +1451,10 @@ mod tests {
// Basically test that our validation tests don't have false-positives
#[test]
fn test_valid_assets() {
let loadouts = assets::load_dir::<LoadoutSpec>("test.loadout.ok", true)
let loadouts = assets::load_rec_dir::<LoadoutSpec>("test.loadout.ok")
.expect("failed to load loadout directory");
for loadout_id in loadouts.ids() {
for loadout_id in loadouts.read().ids() {
let loadout =
LoadoutSpec::load_cloned(loadout_id).expect("failed to load loadout asset");
loadout

4
common/src/comp/inventory/trade_pricing.rs
View File

@ -12,7 +12,7 @@ use crate::{
recipe::{default_component_recipe_book, default_recipe_book, RecipeInput},
trade::Good,
};
use assets::AssetGuard;
use assets::AssetReadGuard;
use hashbrown::HashMap;
use lazy_static::lazy_static;
use serde::Deserialize;
@ -528,7 +528,7 @@ struct RememberedRecipe {
input: Vec<(ItemDefinitionIdOwned, u32)>,
}
fn get_scaling(contents: &AssetGuard<TradingPriceFile>, good: Good) -> f32 {
fn get_scaling(contents: &AssetReadGuard<TradingPriceFile>, good: Good) -> f32 {
contents
.good_scaling
.iter()

4
common/src/generation.rs
View File

@ -165,8 +165,8 @@ impl EntityConfig {
/// Return all entity config specifiers
pub fn try_all_entity_configs() -> Result<Vec<String>, Error> {
let configs = assets::load_dir::<EntityConfig>("common.entity", true)?;
Ok(configs.ids().map(|id| id.to_string()).collect())
let configs = assets::load_rec_dir::<EntityConfig>("common.entity")?;
Ok(configs.read().ids().map(|id| id.to_string()).collect())
}
#[derive(Clone)]

16
common/src/lottery.rs
View File

@ -370,6 +370,7 @@ impl Default for LootSpec<String> {
pub mod tests {
use super::*;
use crate::{assets, comp::Item};
use assets::AssetExt;
#[cfg(test)]
pub fn validate_loot_spec(item: &LootSpec<String>) {
@ -379,8 +380,8 @@ pub mod tests {
Item::new_from_asset_expect(item);
},
LootSpec::LootTable(loot_table) => {
let loot_table = Lottery::<LootSpec<String>>::load_expect_cloned(loot_table);
validate_table_contents(loot_table);
let loot_table = Lottery::<LootSpec<String>>::load_expect(loot_table).read();
validate_table_contents(&loot_table);
},
LootSpec::Nothing => {},
LootSpec::ModularWeapon {
@ -428,7 +429,7 @@ pub mod tests {
}
}
fn validate_table_contents(table: Lottery<LootSpec<String>>) {
fn validate_table_contents(table: &Lottery<LootSpec<String>>) {
for (_, item) in table.iter() {
validate_loot_spec(item);
}
@ -436,10 +437,11 @@ pub mod tests {
#[test]
fn test_loot_tables() {
let loot_tables =
assets::read_expect_dir::<Lottery<LootSpec<String>>>("common.loot_tables", true);
for loot_table in loot_tables {
validate_table_contents(loot_table.clone());
let loot_tables = assets::load_rec_dir::<Lottery<LootSpec<String>>>("common.loot_tables")
.expect("load loot_tables");
for loot_table in loot_tables.read().ids() {
let loot_table = Lottery::<LootSpec<String>>::load_expect(loot_table);
validate_table_contents(&loot_table.read());
}
}

7
common/src/skillset_builder.rs
View File

@ -160,8 +160,11 @@ mod tests {
#[test]
fn test_all_skillset_assets() {
let skillsets = assets::read_expect_dir::<SkillSetTree>("common.skillset", true);
for skillset in skillsets {
let skillsets =
assets::load_rec_dir::<SkillSetTree>("common.skillset").expect("load skillsets");
for skillset in skillsets.read().ids() {
let skillset = SkillSetTree::load_expect(skillset).read();
drop({
let mut skillset_builder = SkillSetBuilder::default();
let nodes = &*skillset.0;

2
voxygen/Cargo.toml
View File

@ -98,7 +98,7 @@ server = { package = "veloren-server", path = "../server", optional = true, defa
clap = { workspace = true }
# Utility
assets_manager = {version = "0.10", features = ["ab_glyph"]}
assets_manager = {version = "0.11", features = ["ab_glyph"]}
backtrace = "0.3.40"
chrono = { workspace = true }
chumsky = "0.9"

8
world/src/layer/shrub.rs
View File

@ -74,10 +74,10 @@ pub fn apply_shrubs_to(canvas: &mut Canvas, _dynamic_rng: &mut impl Rng) {
let units = UnitChooser::new(shrub.seed).get(shrub.seed).into();
let shrubs = match shrub.kind {
ForestKind::Mangrove => &*JUNGLE_SHRUBS,
ForestKind::Acacia | ForestKind::Baobab => &*SAVANNAH_SHRUBS,
ForestKind::Oak | ForestKind::Chestnut => &*TEMPERATE_SHRUBS,
ForestKind::Pine => &*TAIGA_SHRUBS,
ForestKind::Mangrove => *JUNGLE_SHRUBS,
ForestKind::Acacia | ForestKind::Baobab => *SAVANNAH_SHRUBS,
ForestKind::Oak | ForestKind::Chestnut => *TEMPERATE_SHRUBS,
ForestKind::Pine => *TAIGA_SHRUBS,
_ => continue, // TODO: Add more shrub varieties
}
.read();