diff --git a/.gitlab/CI/build.gitlab-ci.yml b/.gitlab/CI/build.gitlab-ci.yml index e544c4e7ee..cc6fcbd679 100644 --- a/.gitlab/CI/build.gitlab-ci.yml +++ b/.gitlab/CI/build.gitlab-ci.yml @@ -6,7 +6,7 @@ unittests: script: - ln -s /dockercache/cache-all target - rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly - - cargo test --package veloren-voxygen --lib test_all_localizations -- --nocapture --ignored + - cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored - rm -r target/debug/incremental* || echo "all good" # TMP FIX FOR 2021-03-22-nightly - cargo test retry: diff --git a/Cargo.lock b/Cargo.lock index 385f69d304..c3cf499305 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5514,7 +5514,6 @@ name = "veloren-common" version = "0.9.0" dependencies = [ "approx 0.4.0", - "assets_manager", "bitflags", "criterion", "crossbeam-channel", @@ -5523,7 +5522,6 @@ dependencies = [ "dot_vox", "enum-iterator", "hashbrown", - "image", "indexmap", "lazy_static", "num-derive", @@ -5531,10 +5529,8 @@ dependencies = [ "ordered-float 2.1.1", "rand 0.8.3", "rayon", - "ron", "roots", "serde", - "serde_json", "serde_repr", "slab", "slotmap", @@ -5548,9 +5544,22 @@ dependencies = [ "tracing-subscriber", "uuid", "vek", + "veloren-common-assets", "veloren-common-base", ] +[[package]] +name = "veloren-common-assets" +version = "0.9.0" +dependencies = [ + "assets_manager", + "dot_vox", + "image", + "lazy_static", + "ron", + "tracing", +] + [[package]] name = "veloren-common-base" version = "0.9.0" @@ -5643,6 +5652,19 @@ dependencies = [ "veloren-common-net", ] +[[package]] +name = "veloren-i18n" +version = "0.9.0" +dependencies = [ + "deunicode", + "git2", + "hashbrown", + "ron", + "serde", + "tracing", + "veloren-common-assets", +] + [[package]] name = "veloren-network" version = "0.3.0" @@ -5795,7 +5817,6 @@ dependencies = [ "cpal", "criterion", "crossbeam", - "deunicode", "directories-next", "dispatch 0.1.4", "dot_vox", @@ -5805,7 +5826,6 @@ dependencies = [ "gfx_device_gl", "gfx_gl", "gilrs", - "git2", "glsl-include", "glutin", "glyph_brush", @@ -5843,6 +5863,7 @@ dependencies = [ "veloren-common-net", "veloren-common-state", "veloren-common-systems", + "veloren-i18n", "veloren-server", "veloren-voxygen-anim", "veloren-world", diff --git a/Cargo.toml b/Cargo.toml index 5e1c09240b..90fc87e346 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ cargo-features = ["named-profiles","profile-overrides"] [workspace] members = [ "common", + "common/assets", "common/base", "common/ecs", "common/net", @@ -18,9 +19,10 @@ members = [ "voxygen", "voxygen/anim", "voxygen/anim/dyn", + "voxygen/i18n", "world", "network", - "network/protocol" + "network/protocol", ] # default profile for devs, fast to compile, okay enough to run, no debug information diff --git a/common/Cargo.toml b/common/Cargo.toml index 2d8f2ed24e..30c25accd0 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -39,14 +39,10 @@ uuid = { version = "0.8.1", default-features = false, features = ["serde", "v4"] rand = "0.8" # Assets +common-assets = {package = "veloren-common-assets", path = "assets"} dot_vox = "4.0" -image = { version = "0.23.12", default-features = false, features = ["png"] } # Assets -assets_manager = {version = "0.4.2", features = ["bincode", "ron", "json", "hot-reloading"]} -# Serde -ron = { version = "0.6", default-features = false } -serde_json = "1.0.50" serde_repr = "0.1.6" # csv export diff --git a/common/assets/Cargo.toml b/common/assets/Cargo.toml new file mode 100644 index 0000000000..6da6273516 --- /dev/null +++ b/common/assets/Cargo.toml @@ -0,0 +1,14 @@ +[package] +authors = ["juliancoffee ", "Marcel Märtens "] +edition = "2018" +name = "veloren-common-assets" +description = "Crate for game loading assets for veloren." +version = "0.9.0" + +[dependencies] +lazy_static = "1.4.0" +assets_manager = {version = "0.4.2", features = ["bincode", "ron", "json", "hot-reloading"]} +ron = { version = "0.6", default-features = false } +dot_vox = "4.0" +image = { version = "0.23.12", default-features = false, features = ["png"] } +tracing = "0.1" diff --git a/common/src/assets.rs b/common/assets/src/lib.rs similarity index 56% rename from common/src/assets.rs rename to common/assets/src/lib.rs index 737810da46..4640362147 100644 --- a/common/src/assets.rs +++ b/common/assets/src/lib.rs @@ -20,7 +20,8 @@ pub use assets_manager::{ lazy_static! { /// The HashMap where all loaded assets are stored in. - static ref ASSETS: AssetCache = AssetCache::new(&*ASSETS_PATH).unwrap(); + static ref ASSETS: AssetCache = + AssetCache::new(&*ASSETS_PATH).unwrap(); } pub fn start_hot_reloading() { ASSETS.enhance_hot_reloading(); } @@ -35,9 +36,9 @@ pub trait AssetExt: Sized + Send + Sync + 'static { /// Function used to load assets from the filesystem or the cache. /// Example usage: /// ```no_run - /// use veloren_common::assets::{self, AssetExt}; + /// use veloren_common_assets::{AssetExt, Image}; /// - /// let my_image = assets::Image::load("core.ui.backgrounds.city").unwrap(); + /// let my_image = Image::load("core.ui.backgrounds.city").unwrap(); /// ``` fn load(specifier: &str) -> Result, Error>; @@ -53,9 +54,9 @@ pub trait AssetExt: Sized + Send + Sync + 'static { /// Function used to load essential assets from the filesystem or the cache. /// It will panic if the asset is not found. Example usage: /// ```no_run - /// use veloren_common::assets::{self, AssetExt}; + /// use veloren_common_assets::{AssetExt, Image}; /// - /// let my_image = assets::Image::load_expect("core.ui.backgrounds.city"); + /// let my_image = Image::load_expect("core.ui.backgrounds.city"); /// ``` #[track_caller] fn load_expect(specifier: &str) -> AssetHandle { @@ -134,6 +135,7 @@ lazy_static! { /// 3. Download & hopefully extract zip (`assets` next to binary) /// 4. Running through cargo (`assets` in workspace root but not always in cwd incase you `cd voxygen && cargo r`) /// 5. Running executable in the target dir (`assets` in workspace) + /// 6. Running tests (`assets` in workspace root) pub static ref ASSETS_PATH: PathBuf = { let mut paths = Vec::new(); @@ -150,19 +152,23 @@ lazy_static! { paths.push(path); } - // 3. Working path + // 3. Root of the repository if let Ok(path) = std::env::current_dir() { - paths.push(path); + // If we are in the root, push path + if path.join(".git").is_dir() { + paths.push(path); + } else { + // Search .git directory in parent directries + for ancestor in path.ancestors().take(10) { + if ancestor.join(".git").is_dir() { + paths.push(ancestor.to_path_buf()); + break; + } + } + } } - // 4. Cargo Workspace (e.g. local development) - // https://github.com/rust-lang/cargo/issues/3946#issuecomment-359619839 - if let Ok(Ok(path)) = std::env::var("CARGO_MANIFEST_DIR").map(|s| s.parse::()) { - paths.push(path.parent().unwrap().to_path_buf()); - paths.push(path); - } - - // 5. System paths + // 4. System paths #[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))] { if let Ok(result) = std::env::var("XDG_DATA_HOME") { @@ -247,112 +253,3 @@ impl Compound for Directory { Ok(Directory(files)) } } - -#[cfg(test)] -mod tests { - #[test] - fn test_assets_items() { - // TODO: Figure out how to get file name in error so only a single glob is - // needed - - // Separated out into subsections so that error more descriptive - crate::comp::item::Item::new_from_asset_glob("common.items.armor.*") - .expect("Failed to iterate over armors."); - - crate::comp::item::Item::new_from_asset_glob("common.items.boss_drops.*") - .expect("Failed to iterate over boss drops."); - - crate::comp::item::Item::new_from_asset_glob("common.items.consumable.*") - .expect("Failed to iterate over consumables."); - - crate::comp::item::Item::new_from_asset_glob("common.items.crafting_ing.*") - .expect("Failed to iterate over crafting ingredients."); - - crate::comp::item::Item::new_from_asset_glob("common.items.crafting_tools.*") - .expect("Failed to iterate over crafting tools."); - - crate::comp::item::Item::new_from_asset_glob("common.items.debug.*") - .expect("Failed to iterate over debug items."); - - crate::comp::item::Item::new_from_asset_glob("common.items.flowers.*") - .expect("Failed to iterate over flower items."); - - crate::comp::item::Item::new_from_asset_glob("common.items.food.*") - .expect("Failed to iterate over food items."); - - crate::comp::item::Item::new_from_asset_glob("common.items.glider.*") - .expect("Failed to iterate over gliders."); - - crate::comp::item::Item::new_from_asset_glob("common.items.grasses.*") - .expect("Failed to iterate over grasses."); - - crate::comp::item::Item::new_from_asset_glob("common.items.lantern.*") - .expect("Failed to iterate over lanterns."); - - crate::comp::item::Item::new_from_asset_glob("common.items.npc_armor.*") - .expect("Failed to iterate over npc armors."); - - crate::comp::item::Item::new_from_asset_glob("common.items.npc_weapons.*") - .expect("Failed to iterate over npc weapons."); - - crate::comp::item::Item::new_from_asset_glob("common.items.ore.*") - .expect("Failed to iterate over ores."); - - crate::comp::item::Item::new_from_asset_glob("common.items.tag_examples.*") - .expect("Failed to iterate over tag examples."); - - crate::comp::item::Item::new_from_asset_glob("common.items.testing.*") - .expect("Failed to iterate over testing items."); - - crate::comp::item::Item::new_from_asset_glob("common.items.utility.*") - .expect("Failed to iterate over utility items."); - - // Checks each weapon type to allow errors to be located more easily - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.axe.*") - .expect("Failed to iterate over axes."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.axe_1h.*") - .expect("Failed to iterate over 1h axes."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.bow.*") - .expect("Failed to iterate over bows."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.dagger.*") - .expect("Failed to iterate over daggers."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.empty.*") - .expect("Failed to iterate over empty."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.hammer.*") - .expect("Failed to iterate over hammers."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.hammer_1h.*") - .expect("Failed to iterate over 1h hammers."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sceptre.*") - .expect("Failed to iterate over sceptres."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.shield.*") - .expect("Failed to iterate over shields."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.staff.*") - .expect("Failed to iterate over staffs."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sword.*") - .expect("Failed to iterate over swords."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sword_1h.*") - .expect("Failed to iterate over 1h swords."); - - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.tool.*") - .expect("Failed to iterate over tools."); - - // Checks all weapons should more weapons be added later - crate::comp::item::Item::new_from_asset_glob("common.items.weapons.*") - .expect("Failed to iterate over weapons."); - - // Final at the end to account for a new folder being added - crate::comp::item::Item::new_from_asset_glob("common.items.*") - .expect("Failed to iterate over item folders."); - } -} diff --git a/common/src/comp/inventory/item/mod.rs b/common/src/comp/inventory/item/mod.rs index c7d5d4f841..83b07c9e0b 100644 --- a/common/src/comp/inventory/item/mod.rs +++ b/common/src/comp/inventory/item/mod.rs @@ -407,8 +407,8 @@ impl Deref for Item { } impl assets::Compound for ItemDef { - fn load( - cache: &assets_manager::AssetCache, + fn load( + cache: &assets::AssetCache, specifier: &str, ) -> Result { // load from the filesystem first, but if the file doesn't exist, see if it's a @@ -853,3 +853,111 @@ impl<'a, T: ItemDesc + ?Sized> ItemDesc for &'a T { fn tags(&self) -> &[ItemTag] { (*self).tags() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_assets_items() { + // TODO: Figure out how to get file name in error so only a single glob is + // needed + + // Separated out into subsections so that error more descriptive + Item::new_from_asset_glob("common.items.armor.*").expect("Failed to iterate over armors."); + + Item::new_from_asset_glob("common.items.boss_drops.*") + .expect("Failed to iterate over boss drops."); + + Item::new_from_asset_glob("common.items.consumable.*") + .expect("Failed to iterate over consumables."); + + Item::new_from_asset_glob("common.items.crafting_ing.*") + .expect("Failed to iterate over crafting ingredients."); + + Item::new_from_asset_glob("common.items.crafting_tools.*") + .expect("Failed to iterate over crafting tools."); + + Item::new_from_asset_glob("common.items.debug.*") + .expect("Failed to iterate over debug items."); + + Item::new_from_asset_glob("common.items.flowers.*") + .expect("Failed to iterate over flower items."); + + Item::new_from_asset_glob("common.items.food.*") + .expect("Failed to iterate over food items."); + + Item::new_from_asset_glob("common.items.glider.*") + .expect("Failed to iterate over gliders."); + + Item::new_from_asset_glob("common.items.grasses.*") + .expect("Failed to iterate over grasses."); + + Item::new_from_asset_glob("common.items.lantern.*") + .expect("Failed to iterate over lanterns."); + + Item::new_from_asset_glob("common.items.npc_armor.*") + .expect("Failed to iterate over npc armors."); + + Item::new_from_asset_glob("common.items.npc_weapons.*") + .expect("Failed to iterate over npc weapons."); + + Item::new_from_asset_glob("common.items.ore.*").expect("Failed to iterate over ores."); + + Item::new_from_asset_glob("common.items.tag_examples.*") + .expect("Failed to iterate over tag examples."); + + Item::new_from_asset_glob("common.items.testing.*") + .expect("Failed to iterate over testing items."); + + Item::new_from_asset_glob("common.items.utility.*") + .expect("Failed to iterate over utility items."); + + // Checks each weapon type to allow errors to be located more easily + Item::new_from_asset_glob("common.items.weapons.axe.*") + .expect("Failed to iterate over axes."); + + Item::new_from_asset_glob("common.items.weapons.axe_1h.*") + .expect("Failed to iterate over 1h axes."); + + Item::new_from_asset_glob("common.items.weapons.bow.*") + .expect("Failed to iterate over bows."); + + Item::new_from_asset_glob("common.items.weapons.dagger.*") + .expect("Failed to iterate over daggers."); + + Item::new_from_asset_glob("common.items.weapons.empty.*") + .expect("Failed to iterate over empty."); + + Item::new_from_asset_glob("common.items.weapons.hammer.*") + .expect("Failed to iterate over hammers."); + + Item::new_from_asset_glob("common.items.weapons.hammer_1h.*") + .expect("Failed to iterate over 1h hammers."); + + Item::new_from_asset_glob("common.items.weapons.sceptre.*") + .expect("Failed to iterate over sceptres."); + + Item::new_from_asset_glob("common.items.weapons.shield.*") + .expect("Failed to iterate over shields."); + + Item::new_from_asset_glob("common.items.weapons.staff.*") + .expect("Failed to iterate over staffs."); + + Item::new_from_asset_glob("common.items.weapons.sword.*") + .expect("Failed to iterate over swords."); + + Item::new_from_asset_glob("common.items.weapons.sword_1h.*") + .expect("Failed to iterate over 1h swords."); + + Item::new_from_asset_glob("common.items.weapons.tool.*") + .expect("Failed to iterate over tools."); + + // Checks all weapons should more weapons be added later + Item::new_from_asset_glob("common.items.weapons.*") + .expect("Failed to iterate over weapons."); + + // Final at the end to account for a new folder being added + Item::new_from_asset_glob("common.items.*").expect("Failed to iterate over item folders."); + } +} diff --git a/common/src/comp/inventory/item/tool.rs b/common/src/comp/inventory/item/tool.rs index 3f4053b9ad..50f62365d5 100644 --- a/common/src/comp/inventory/item/tool.rs +++ b/common/src/comp/inventory/item/tool.rs @@ -392,8 +392,8 @@ impl Asset for AbilityMap { } impl assets::Compound for AbilityMap { - fn load( - cache: &assets_manager::AssetCache, + fn load( + cache: &assets::AssetCache, specifier: &str, ) -> Result { let manifest = cache.load::>(specifier)?.read(); diff --git a/common/src/comp/inventory/trade_pricing.rs b/common/src/comp/inventory/trade_pricing.rs index e1ba75bd74..78b9fad9b6 100644 --- a/common/src/comp/inventory/trade_pricing.rs +++ b/common/src/comp/inventory/trade_pricing.rs @@ -4,7 +4,7 @@ use crate::{ recipe::{default_recipe_book, RecipeInput}, trade::Good, }; -use assets_manager::AssetGuard; +use assets::AssetGuard; use hashbrown::HashMap; use lazy_static::lazy_static; use serde::Deserialize; diff --git a/common/src/lib.rs b/common/src/lib.rs index a3792863a7..b561bf0720 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -23,7 +23,7 @@ pub use uuid; // modules #[cfg(not(target_arch = "wasm32"))] -pub mod assets; +pub use common_assets as assets; #[cfg(not(target_arch = "wasm32"))] pub mod astar; #[cfg(not(target_arch = "wasm32"))] mod cached_spatial_grid; diff --git a/common/src/recipe.rs b/common/src/recipe.rs index 68af4728b0..31bf2cfbfd 100644 --- a/common/src/recipe.rs +++ b/common/src/recipe.rs @@ -108,10 +108,10 @@ impl assets::Asset for RawRecipeBook { } impl assets::Compound for RecipeBook { - fn load( - cache: &assets_manager::AssetCache, + fn load( + cache: &assets::AssetCache, specifier: &str, - ) -> Result { + ) -> Result { #[inline] fn load_item_def(spec: &(String, u32)) -> Result<(Arc, u32), assets::Error> { let def = Arc::::load_cloned(&spec.0)?; diff --git a/common/src/terrain/structure.rs b/common/src/terrain/structure.rs index c9b12d8d0c..14a658b43a 100644 --- a/common/src/terrain/structure.rs +++ b/common/src/terrain/structure.rs @@ -60,8 +60,8 @@ impl std::ops::Deref for StructuresGroup { } impl assets::Compound for StructuresGroup { - fn load( - cache: &assets_manager::AssetCache, + fn load( + cache: &assets::AssetCache, specifier: &str, ) -> Result { let specs = cache.load::(specifier)?.read(); @@ -118,8 +118,8 @@ impl ReadVol for Structure { } impl assets::Compound for BaseStructure { - fn load( - cache: &assets_manager::AssetCache, + fn load( + cache: &assets::AssetCache, specifier: &str, ) -> Result { let dot_vox_data = cache.load::(specifier)?.read(); diff --git a/voxygen/Cargo.toml b/voxygen/Cargo.toml index 80787cdea2..47e5be9c1a 100644 --- a/voxygen/Cargo.toml +++ b/voxygen/Cargo.toml @@ -42,6 +42,7 @@ common-systems = {package = "veloren-common-systems", path = "../common/systems" common-state = {package = "veloren-common-state", path = "../common/state"} anim = {package = "veloren-voxygen-anim", path = "anim"} +i18n = {package = "veloren-i18n", path = "i18n"} # Graphics gfx = "0.18.2" @@ -81,7 +82,6 @@ chrono = "0.4.9" cpal = "0.13" copy_dir = "0.1.2" crossbeam = "0.8.0" -deunicode = "1.0" # TODO: remove directories-next = "2.0" dot_vox = "4.0" @@ -121,7 +121,6 @@ winres = "0.1" [dev-dependencies] criterion = "0.3" -git2 = "0.13" world = {package = "veloren-world", path = "../world"} rayon = "1.5.0" diff --git a/voxygen/i18n/Cargo.toml b/voxygen/i18n/Cargo.toml new file mode 100644 index 0000000000..e1b60890da --- /dev/null +++ b/voxygen/i18n/Cargo.toml @@ -0,0 +1,21 @@ +[package] +authors = ["juliancoffee ", "Rémy Phelipot"] +edition = "2018" +name = "veloren-i18n" +description = "Crate for internalization and diagnostic of existing localizations." +version = "0.9.0" + +[[bin]] +name = "i18n-check" + +[dependencies] +# Assets +hashbrown = { version = "0.9", features = ["serde", "nightly"] } +common-assets = {package = "veloren-common-assets", path = "../../common/assets"} +deunicode = "1.0" +serde = { version = "1.0", features = ["derive"] } +# Diagnostic +git2 = "0.13" +ron = "0.6" + +tracing = "0.1" diff --git a/voxygen/i18n/README.md b/voxygen/i18n/README.md new file mode 100644 index 0000000000..06704a4f36 --- /dev/null +++ b/voxygen/i18n/README.md @@ -0,0 +1,8 @@ +# Usage +Get diagnostic for specific language
+`$ cargo run --bin i18n-check -- --lang `
+Test all languages
+`$ cargo run --bin i18n-check -- --all` +Verify all directories
+`$ cargo run --bin i18n-check -- --verify` + diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs new file mode 100644 index 0000000000..d7cdf7656d --- /dev/null +++ b/voxygen/i18n/src/analysis.rs @@ -0,0 +1,777 @@ +use ron::de::{from_bytes, from_reader}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use hashbrown::{HashMap, HashSet}; + +/// The reference language, aka the more up-to-date localization data. Also the +/// default language at first startup. +const REFERENCE_LANG: &str = "en"; + +const LANG_MANIFEST_FILE: &str = "_manifest"; + +/// How a language can be described +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct LanguageMetadata { + /// A human friendly language name (e.g. "English (US)") + language_name: String, + + /// A short text identifier for this language (e.g. "en_US") + /// + /// On the opposite of `language_name` that can change freely, + /// `language_identifier` value shall be stable in time as it + /// is used by setting components to store the language + /// selected by the user. + language_identifier: String, +} + +/// Store font metadata +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +struct Font { + /// Key to retrieve the font in the asset system + asset_key: String, + + /// Scale ratio to resize the UI text dynamicly + scale_ratio: f32, +} + +/// Store font metadata +type Fonts = HashMap; + +/// Raw localization data, expect the strings to not be loaded here +/// However, metadata informations are correct +/// See `Localization` for more info on each attributes +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +struct RawLocalization { + sub_directories: Vec, + string_map: HashMap, + vector_map: HashMap>, + convert_utf8_to_ascii: bool, + fonts: Fonts, + metadata: LanguageMetadata, +} + +/// Store internationalization data +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Localization { + /// A list of subdirectories to lookup for localization files + sub_directories: Vec, + + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + vector_map: HashMap>, + + /// Whether to convert the input text encoded in UTF-8 + /// into a ASCII version by using the `deunicode` crate. + convert_utf8_to_ascii: bool, + + /// Font configuration is stored here + fonts: Fonts, + + metadata: LanguageMetadata, +} + +/// Store internationalization maps +/// These structs are meant to be merged into a Localization +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct LocalizationFragment { + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + vector_map: HashMap>, +} + +impl Localization {} + +impl From for Localization { + fn from(raw: RawLocalization) -> Self { + Self { + sub_directories: raw.sub_directories, + string_map: raw.string_map, + vector_map: raw.vector_map, + convert_utf8_to_ascii: raw.convert_utf8_to_ascii, + fonts: raw.fonts, + metadata: raw.metadata, + } + } +} +impl From for LocalizationFragment { + fn from(raw: RawLocalization) -> Self { + Self { + string_map: raw.string_map, + vector_map: raw.vector_map, + } + } +} + +#[derive(Clone, Debug)] +struct LocalizationList(Vec); + +/// List localization directories as a PathBuf vector +fn i18n_directories(i18n_dir: &Path) -> Vec { + fs::read_dir(i18n_dir) + .unwrap() + .map(|res| res.map(|e| e.path()).unwrap()) + .filter(|e| e.is_dir()) + .collect() +} + +#[derive(Eq, Hash, Debug, PartialEq)] +enum LocalizationState { + UpToDate, + NotFound, + Outdated, + Unknown, + Unused, +} + +#[derive(Debug, PartialEq)] +struct FindLocalization { + uptodate_entries: usize, + outdated_entries: usize, + unused_entries: usize, + notfound_entries: usize, + errors: usize, + real_entry_count: usize, +} + +#[derive(Debug)] +struct LocalizationEntryState { + key_line: Option, + chuck_line_range: Option<(usize, usize)>, + commit_id: Option, + state: LocalizationState, +} + +impl LocalizationEntryState { + fn new() -> LocalizationEntryState { + LocalizationEntryState { + key_line: None, + chuck_line_range: None, + commit_id: None, + state: LocalizationState::Unknown, + } + } +} + +/// Returns the Git blob associated with the given reference and path +fn read_file_from_path<'a>( + repo: &'a git2::Repository, + reference: &git2::Reference, + path: &std::path::Path, +) -> git2::Blob<'a> { + let tree = reference + .peel_to_tree() + .expect("Impossible to peel HEAD to a tree object"); + tree.get_path(path) + .unwrap_or_else(|_| { + panic!( + "Impossible to find the file {:?} in reference {:?}", + path, + reference.name() + ) + }) + .to_object(&repo) + .unwrap() + .peel_to_blob() + .expect("Impossible to fetch the Git object") +} + +fn correspond(line: &str, key: &str) -> bool { + let pat = { + // Get left part of split + let mut begin = line + .split(':') + .next() + .expect("split always produces value") + .trim() + .chars(); + // Remove quotes + begin.next(); + begin.next_back(); + begin.as_str() + }; + + pat == key +} + +fn generate_key_version<'a>( + repo: &'a git2::Repository, + localization: &LocalizationFragment, + path: &std::path::Path, + file_blob: &git2::Blob, +) -> HashMap { + let mut keys: HashMap = localization + .string_map + .keys() + .map(|k| (k.to_owned(), LocalizationEntryState::new())) + .collect(); + let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); + // Find key start lines + let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); + + // Make the file hot + for (line_nb, line) in file_content.lines().enumerate() { + let mut found_key = None; + + for key in to_process.iter() { + if correspond(line, key) { + found_key = Some(key.to_owned()); + } + } + + if let Some(key) = found_key { + keys.get_mut(key).unwrap().key_line = Some(line_nb); + to_process.remove(key); + }; + } + + let mut error_check_set: Vec = vec![]; + // Find commit for each keys + repo.blame_file(path, None) + .expect("Impossible to generate the Git blame") + .iter() + .for_each(|e: git2::BlameHunk| { + for (key, state) in keys.iter_mut() { + let line = match state.key_line { + Some(l) => l, + None => { + if !error_check_set.contains(key) { + eprintln!( + "Key {} does not have a git line in it's state! Skipping key.", + key + ); + error_check_set.push(key.clone()); + } + continue; + }, + }; + + if line + 1 >= e.final_start_line() + && line + 1 < e.final_start_line() + e.lines_in_hunk() + { + state.chuck_line_range = Some(( + e.final_start_line(), + e.final_start_line() + e.lines_in_hunk(), + )); + state.commit_id = match state.commit_id { + Some(existing_commit) => { + match repo.graph_descendant_of(e.final_commit_id(), existing_commit) { + Ok(true) => Some(e.final_commit_id()), + Ok(false) => Some(existing_commit), + Err(err) => panic!("{}", err), + } + }, + None => Some(e.final_commit_id()), + }; + } + } + }); + + keys +} + +fn complete_key_versions<'a>( + repo: &'a git2::Repository, + head_ref: &git2::Reference, + i18n_key_versions: &mut HashMap, + root_dir: &Path, + asset_path: &Path, +) { + //TODO: review unwraps in this file + + // For each file (if it's not a directory) in directory + for i18n_file in root_dir.join(&asset_path).read_dir().unwrap().flatten() { + if let Ok(file_type) = i18n_file.file_type() { + if file_type.is_file() { + println!("-> {:?}", i18n_file.file_name()); + + let full_path = i18n_file.path(); + let path = full_path.strip_prefix(root_dir).unwrap(); + let i18n_blob = read_file_from_path(&repo, &head_ref, &path); + let i18n: LocalizationFragment = match from_bytes(i18n_blob.content()) { + Ok(v) => v, + Err(e) => { + eprintln!( + "Could not parse {} RON file, skipping: {}", + i18n_file.path().to_string_lossy(), + e + ); + continue; + }, + }; + i18n_key_versions.extend(generate_key_version(&repo, &i18n, &path, &i18n_blob)); + } + } + } +} + +fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { + // Walk through each file in the directory + for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() { + if let Ok(file_type) = i18n_file.file_type() { + // Skip folders and the manifest file (which does not contain the same struct we + // want to load) + if file_type.is_file() + && i18n_file.file_name().to_string_lossy() + != (LANG_MANIFEST_FILE.to_string() + ".ron") + { + let full_path = i18n_file.path(); + println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap()); + let f = fs::File::open(&full_path).expect("Failed opening file"); + let _: LocalizationFragment = match from_reader(f) { + Ok(v) => v, + Err(e) => { + panic!( + "Could not parse {} RON file, error: {}", + full_path.to_string_lossy(), + e + ); + }, + }; + } + } + } +} + +/// Test to verify all languages that they are VALID and loadable, without +/// need of git just on the local assets folder +/// `root_dir` - absolute path to main repo +/// `asset_path` - relative path to asset directory (right now it is +/// 'assets/voxygen/i18n') +pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { + let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG); + let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + assert!( + root_dir.join(&ref_i18n_dir_path).is_dir(), + "Reference language folder doesn't exist, something is wrong!" + ); + assert!( + root_dir.join(&ref_i18n_path).is_file(), + "Reference language manifest file doesn't exist, something is wrong!" + ); + let i18n_directories = i18n_directories(&root_dir.join(asset_path)); + // This simple check ONLY guarantees that an arbitrary minimum of translation + // files exists. It's just to notice unintentional deletion of all + // files, or modifying the paths. In case you want to delete all + // language you have to adjust this number: + assert!( + i18n_directories.len() > 5, + "have less than 5 translation folders, arbitrary minimum check failed. Maybe the i18n \ + folder is empty?" + ); + for i18n_directory in i18n_directories { + // Attempt to load the manifest file + let manifest_path = i18n_directory.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + println!( + "verifying {:?}", + manifest_path.strip_prefix(&root_dir).unwrap() + ); + let f = fs::File::open(&manifest_path).expect("Failed opening file"); + let raw_localization: RawLocalization = match from_reader(f) { + Ok(v) => v, + Err(e) => { + panic!( + "Could not parse {} RON file, error: {}", + i18n_directory.to_string_lossy(), + e + ); + }, + }; + // Walk through each files and try to load them + verify_localization_directory(root_dir, &i18n_directory); + // Walk through each subdirectories and try to load files in them + for sub_directory in raw_localization.sub_directories.iter() { + let subdir_path = &i18n_directory.join(sub_directory); + verify_localization_directory(root_dir, &subdir_path); + } + } +} + +// Helper function to test localization directory +// `asset_path` - path to localization directory. Relative from root of the +// repo. +// `root_dir` - absolute path to repo +// `ref_i18n_path` - path to reference manifest +// `i18n_references` - keys from reference language +// `repo` - git object for main repo +// `head_ref` - HEAD +fn test_localization_directory( + asset_path: &Path, + root_dir: &Path, + ref_i18n_path: &Path, + i18n_references: &HashMap, + repo: &git2::Repository, + head_ref: &git2::Reference, +) -> Option { + let relfile = asset_path.join(&(LANG_MANIFEST_FILE.to_string() + ".ron")); + if relfile == ref_i18n_path { + return None; + } + println!("\n-----------------------------------"); + println!("{:?}", relfile); + println!("-----------------------------------"); + + // Find the localization entry state + let current_blob = read_file_from_path(&repo, &head_ref, &relfile); + let current_loc: RawLocalization = match from_bytes(current_blob.content()) { + Ok(v) => v, + Err(e) => { + eprintln!( + "Could not parse {} RON file, skipping: {}", + relfile.to_string_lossy(), + e + ); + return None; + }, + }; + let mut current_i18n = generate_key_version( + &repo, + &LocalizationFragment::from(current_loc.clone()), + &relfile, + ¤t_blob, + ); + // read HEAD for the fragment files + complete_key_versions(&repo, &head_ref, &mut current_i18n, root_dir, &asset_path); + // read HEAD for the subfolders + for sub_directory in current_loc.sub_directories.iter() { + let subdir_path = &asset_path.join(sub_directory); + complete_key_versions(&repo, &head_ref, &mut current_i18n, root_dir, &subdir_path); + } + + for (ref_key, ref_state) in i18n_references.iter() { + match current_i18n.get_mut(ref_key) { + Some(state) => { + let commit_id = match state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in i18n file {} is missing! Skipping key.", + ref_key, + relfile.to_string_lossy() + ); + continue; + }, + }; + let ref_commit_id = match ref_state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in reference i18n file is missing! Skipping key.", + ref_key + ); + continue; + }, + }; + if commit_id != ref_commit_id + && !repo + .graph_descendant_of(commit_id, ref_commit_id) + .unwrap_or(false) + { + state.state = LocalizationState::Outdated; + } else { + state.state = LocalizationState::UpToDate; + } + }, + None => { + current_i18n.insert(ref_key.to_owned(), LocalizationEntryState { + key_line: None, + chuck_line_range: None, + commit_id: None, + state: LocalizationState::NotFound, + }); + }, + } + } + + let ref_keys: HashSet<&String> = i18n_references.keys().collect(); + for (_, state) in current_i18n + .iter_mut() + .filter(|&(k, _)| !ref_keys.contains(k)) + { + state.state = LocalizationState::Unused; + } + + let keys: Vec<&String> = current_i18n.keys().collect(); + let mut state_map: HashMap)>> = + HashMap::new(); + state_map.insert(LocalizationState::Outdated, Vec::new()); + state_map.insert(LocalizationState::NotFound, Vec::new()); + state_map.insert(LocalizationState::Unknown, Vec::new()); + state_map.insert(LocalizationState::Unused, Vec::new()); + + let current_i18n_entry_count = current_i18n.len(); + let mut uptodate_entries = 0; + let mut outdated_entries = 0; + let mut unused_entries = 0; + let mut notfound_entries = 0; + let mut unknown_entries = 0; + + for key in keys { + let entry = current_i18n.get(key).unwrap(); + if entry.state != LocalizationState::UpToDate { + let state_keys = state_map + .get_mut(&entry.state) + .expect("vectors must be added"); + state_keys.push((key, entry.commit_id)); + match entry.state { + LocalizationState::Outdated => outdated_entries += 1, + LocalizationState::NotFound => notfound_entries += 1, + LocalizationState::Unknown => unknown_entries += 1, + LocalizationState::Unused => unused_entries += 1, + LocalizationState::UpToDate => unreachable!(), + }; + } else { + uptodate_entries += 1; + } + } + + // Display + println!( + "\n{:60}| {:40} | {:40}\n", + "Key name", + relfile.to_str().unwrap(), + ref_i18n_path.to_str().unwrap() + ); + + for (state, mut lines) in state_map { + if lines.is_empty() { + continue; + } + println!("\n\t[{:?}]", state); + lines.sort(); + for line in lines { + println!( + "{:60}| {:40} | {:40}", + line.0, + line.1 + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_string()), + i18n_references + .get(line.0) + .map(|s| s.commit_id) + .flatten() + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_string()), + ); + } + } + + println!( + "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", + uptodate_entries, outdated_entries, unused_entries, notfound_entries, unknown_entries + ); + + // Calculate key count that actually matter for the status of the translation + // Unused entries don't break the game + let real_entry_count = current_i18n_entry_count - unused_entries; + let uptodate_percent = (uptodate_entries as f32 / real_entry_count as f32) * 100_f32; + let outdated_percent = (outdated_entries as f32 / real_entry_count as f32) * 100_f32; + let untranslated_percent = + ((notfound_entries + unknown_entries) as f32 / real_entry_count as f32) * 100_f32; + + println!( + "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", + uptodate_percent, outdated_percent, untranslated_percent, + ); + + let result = FindLocalization { + uptodate_entries, + unused_entries, + outdated_entries, + notfound_entries, + errors: unknown_entries, + real_entry_count, + }; + Some(result) +} + +/// Test one language +/// `code` - name of the directory in assets (de_DE for example) +/// `root_dir` - absolute path to main repo +/// `asset_path` - relative path to asset directory (right now it is +/// 'assets/voxygen/i18n') +pub fn test_specific_localization(code: String, root_dir: &Path, asset_path: &Path) { + // Relative paths from root of repo to assets + let ref_lang_dir = asset_path.join(REFERENCE_LANG); + let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + + // Initialize Git objects + let repo = git2::Repository::discover(&root_dir) + .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); + let head_ref = repo.head().expect("Impossible to get the HEAD reference"); + + // Read HEAD for the reference language manifest + let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest); + let loc: RawLocalization = from_bytes(ref_manifest_blob.content()) + .expect("Expect to parse reference i18n RON file, can't proceed without it"); + let mut i18n_references: HashMap = generate_key_version( + &repo, + &LocalizationFragment::from(loc.clone()), + &ref_manifest, + &ref_manifest_blob, + ); + + // Gathering info about keys from reference language + complete_key_versions( + &repo, + &head_ref, + &mut i18n_references, + root_dir, + &ref_lang_dir, + ); + for sub_directory in loc.sub_directories.iter() { + let subdir_path = &ref_lang_dir.join(sub_directory); + complete_key_versions( + &repo, + &head_ref, + &mut i18n_references, + root_dir, + &subdir_path, + ); + } + + // Testing how specific language is localized + let dir = asset_path.join(code); + test_localization_directory( + &dir, + root_dir, + &ref_manifest, + &i18n_references, + &repo, + &head_ref, + ); +} + +/// Test all localizations +/// `root_dir` - absolute path to main repo +/// `asset_path` - relative path to asset directory (right now it is +/// 'assets/voxygen/i18n') +pub fn test_all_localizations(root_dir: &Path, asset_path: &Path) { + let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG); + let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + + if !root_dir.join(&ref_i18n_dir_path).is_dir() { + panic!( + "Reference language folder not found {:?}", + &ref_i18n_dir_path + ) + } + if !root_dir.join(&ref_i18n_path).is_file() { + panic!("Reference language file not found {:?}", &ref_i18n_path) + } + + // Initialize Git objects + let repo = git2::Repository::discover(&root_dir) + .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); + let head_ref = repo.head().expect("Impossible to get the HEAD reference"); + + // Read HEAD for the reference language file + let i18n_ref_blob = read_file_from_path(&repo, &head_ref, &ref_i18n_path); + let loc: RawLocalization = from_bytes(i18n_ref_blob.content()) + .expect("Expect to parse reference i18n RON file, can't proceed without it"); + let mut i18n_references: HashMap = generate_key_version( + &repo, + &LocalizationFragment::from(loc.clone()), + &ref_i18n_path, + &i18n_ref_blob, + ); + + // Gathering info about keys from reference language + complete_key_versions( + &repo, + &head_ref, + &mut i18n_references, + root_dir, + &ref_i18n_dir_path, + ); + // read HEAD for the subfolders + for sub_directory in loc.sub_directories.iter() { + let subdir_path = &ref_i18n_dir_path.join(sub_directory); + complete_key_versions( + &repo, + &head_ref, + &mut i18n_references, + root_dir, + &subdir_path, + ); + } + + // Compare to other reference files + let i18n_directories = i18n_directories(&root_dir.join(asset_path)); + let mut i18n_entry_counts: HashMap = HashMap::new(); + for dir in &i18n_directories { + let rel_dir = dir.strip_prefix(root_dir).unwrap(); + let result = test_localization_directory( + rel_dir, + root_dir, + &ref_i18n_path, + &i18n_references, + &repo, + &head_ref, + ); + if let Some(values) = result { + i18n_entry_counts.insert(dir.clone(), values); + } + } + + let mut overall_uptodate_entry_count = 0; + let mut overall_outdated_entry_count = 0; + let mut overall_untranslated_entry_count = 0; + let mut overall_real_entry_count = 0; + + println!("-----------------------------------------------------------------------------"); + println!("Overall Translation Status"); + println!("-----------------------------------------------------------------------------"); + println!( + "{:12}| {:8} | {:8} | {:8} | {:8} | {:8}", + "", "up-to-date", "outdated", "untranslated", "unused", "errors", + ); + + for (path, test_result) in i18n_entry_counts { + let FindLocalization { + uptodate_entries: uptodate, + outdated_entries: outdated, + unused_entries: unused, + notfound_entries: untranslated, + errors, + real_entry_count: real, + } = test_result; + overall_uptodate_entry_count += uptodate; + overall_outdated_entry_count += outdated; + overall_untranslated_entry_count += untranslated; + overall_real_entry_count += real; + + println!( + "{:12}|{:8} |{:6} |{:8} |{:6} |{:8}", + path.file_name().unwrap().to_string_lossy(), + uptodate, + outdated, + untranslated, + unused, + errors, + ); + } + + println!( + "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", + (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + ); + println!("-----------------------------------------------------------------------------\n"); +} diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs new file mode 100644 index 0000000000..8f225f47ff --- /dev/null +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -0,0 +1,22 @@ +use std::{env::args, path::Path, vec::Vec}; +use veloren_i18n::analysis; + +fn main() { + let cli: Vec = args().collect(); + + // Generate paths + let curr_dir = std::env::current_dir().unwrap(); + let root = curr_dir.parent().unwrap().parent().unwrap(); + let asset_path = Path::new("assets/voxygen/i18n/"); + for (i, arg) in cli.iter().enumerate() { + match arg.as_str() { + "--all" => analysis::test_all_localizations(root, asset_path), + "--verify" => analysis::verify_all_localizations(root, asset_path), + "--lang" => { + let code = cli[i + 1].clone(); + analysis::test_specific_localization(code, root, asset_path); + }, + _ => continue, + } + } +} diff --git a/voxygen/i18n/src/data.rs b/voxygen/i18n/src/data.rs new file mode 100644 index 0000000000..f86e45e649 --- /dev/null +++ b/voxygen/i18n/src/data.rs @@ -0,0 +1,430 @@ +use crate::assets::{self, AssetExt, AssetGuard, AssetHandle}; +use deunicode::deunicode; +use hashbrown::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; +use tracing::warn; + +/// The reference language, aka the more up-to-date localization data. +/// Also the default language at first startup. +pub const REFERENCE_LANG: &str = "en"; + +pub const LANG_MANIFEST_FILE: &str = "_manifest"; + +/// How a language can be described +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LanguageMetadata { + /// A human friendly language name (e.g. "English (US)") + pub language_name: String, + + /// A short text identifier for this language (e.g. "en_US") + /// + /// On the opposite of `language_name` that can change freely, + /// `language_identifier` value shall be stable in time as it + /// is used by setting components to store the language + /// selected by the user. + pub language_identifier: String, +} + +/// Store font metadata +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Font { + /// Key to retrieve the font in the asset system + pub asset_key: String, + + /// Scale ratio to resize the UI text dynamicly + scale_ratio: f32, +} + +impl Font { + /// Scale input size to final UI size + pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } +} + +/// Store font metadata +pub type Fonts = HashMap; + +/// Raw localization data, expect the strings to not be loaded here +/// However, metadata informations are correct +/// See `Language` for more info on each attributes +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawLocalization { + pub(crate) sub_directories: Vec, + pub(crate) string_map: HashMap, + pub(crate) vector_map: HashMap>, + pub(crate) convert_utf8_to_ascii: bool, + pub(crate) fonts: Fonts, + pub(crate) metadata: LanguageMetadata, +} + +/// Store internationalization data +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Language { + /// A list of subdirectories to lookup for localization files + pub(crate) sub_directories: Vec, + + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + pub(crate) string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + pub(crate) vector_map: HashMap>, + + /// Whether to convert the input text encoded in UTF-8 + /// into a ASCII version by using the `deunicode` crate. + pub(crate) convert_utf8_to_ascii: bool, + + /// Font configuration is stored here + pub(crate) fonts: Fonts, + + pub(crate) metadata: LanguageMetadata, +} + +/// Store internationalization maps +/// These structs are meant to be merged into a Language +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct LocalizationFragment { + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + pub(crate) string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + pub(crate) vector_map: HashMap>, +} + +impl Language { + /// Get a localized text from the given key + pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { + self.string_map.get(key).map(|s| s.as_str()) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { + self.vector_map + .get(key) + .map(|v| { + if !v.is_empty() { + Some(v[index as usize % v.len()].as_str()) + } else { + None + } + }) + .flatten() + } +} + +impl Default for Language { + fn default() -> Self { + Self { + sub_directories: Vec::default(), + string_map: HashMap::default(), + vector_map: HashMap::default(), + ..Default::default() + } + } +} + +impl From for Language { + fn from(raw: RawLocalization) -> Self { + Self { + sub_directories: raw.sub_directories, + string_map: raw.string_map, + vector_map: raw.vector_map, + convert_utf8_to_ascii: raw.convert_utf8_to_ascii, + fonts: raw.fonts, + metadata: raw.metadata, + } + } +} +impl From for LocalizationFragment { + fn from(raw: RawLocalization) -> Self { + Self { + string_map: raw.string_map, + vector_map: raw.vector_map, + } + } +} + +impl assets::Asset for RawLocalization { + type Loader = assets::RonLoader; + + const EXTENSION: &'static str = "ron"; +} +impl assets::Asset for LocalizationFragment { + type Loader = assets::RonLoader; + + const EXTENSION: &'static str = "ron"; +} + +impl assets::Compound for Language { + fn load( + cache: &assets::AssetCache, + asset_key: &str, + ) -> Result { + let raw = cache + .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? + .cloned(); + let mut localization = Language::from(raw); + + // Walk through files in the folder, collecting localization fragment to merge + // inside the asked_localization + for localization_asset in cache.load_dir::(asset_key)?.iter() { + localization + .string_map + .extend(localization_asset.read().string_map.clone()); + localization + .vector_map + .extend(localization_asset.read().vector_map.clone()); + } + + // Use the localization's subdirectory list to load fragments from there + for sub_directory in localization.sub_directories.iter() { + for localization_asset in cache + .load_dir::(&[asset_key, ".", sub_directory].concat())? + .iter() + { + localization + .string_map + .extend(localization_asset.read().string_map.clone()); + localization + .vector_map + .extend(localization_asset.read().vector_map.clone()); + } + } + + // Update the text if UTF-8 to ASCII conversion is enabled + if localization.convert_utf8_to_ascii { + for value in localization.string_map.values_mut() { + *value = deunicode(value); + } + + for value in localization.vector_map.values_mut() { + *value = value.iter().map(|s| deunicode(s)).collect(); + } + } + localization.metadata.language_name = deunicode(&localization.metadata.language_name); + + Ok(localization) + } +} + +/// the central data structure to handle localization in veloren +// inherit Copy+Clone from AssetHandle +#[derive(Debug, PartialEq, Copy, Clone)] +pub struct LocalizationHandle { + active: AssetHandle, + fallback: Option>, + pub use_english_fallback: bool, +} + +// RAII guard returned from Localization::read(), resembles AssetGuard +pub struct LocalizationGuard { + active: AssetGuard, + fallback: Option>, +} + +// arbitrary choice to minimize changing all of veloren +pub type Localization = LocalizationGuard; + +impl LocalizationGuard { + /// Get a localized text from the given key + /// + /// First lookup is done in the active language, second in + /// the fallback (if present). + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get<'a>(&'a self, key: &'a str) -> &str { + self.active.get(key).unwrap_or_else(|| { + self.fallback + .as_ref() + .map(|f| f.get(key)) + .flatten() + .unwrap_or(key) + }) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { + self.active.get_variation(key, index).unwrap_or_else(|| { + self.fallback + .as_ref() + .map(|f| f.get_variation(key, index)) + .flatten() + .unwrap_or(key) + }) + } + + /// Return the missing keys compared to the reference language + fn list_missing_entries(&self) -> (HashSet, HashSet) { + if let Some(ref_lang) = &self.fallback { + let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); + let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); + let strings = reference_string_keys + .difference(&string_keys) + .cloned() + .collect(); + + let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); + let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); + let vectors = reference_vector_keys + .difference(&vector_keys) + .cloned() + .collect(); + + (strings, vectors) + } else { + (HashSet::default(), HashSet::default()) + } + } + + /// Log missing entries (compared to the reference language) as warnings + pub fn log_missing_entries(&self) { + let (missing_strings, missing_vectors) = self.list_missing_entries(); + for missing_key in missing_strings { + warn!( + "[{:?}] Missing string key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + for missing_key in missing_vectors { + warn!( + "[{:?}] Missing vector key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + } + + pub fn fonts(&self) -> &Fonts { &self.active.fonts } + + pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } +} + +impl LocalizationHandle { + pub fn set_english_fallback(&mut self, use_english_fallback: bool) { + self.use_english_fallback = use_english_fallback; + } + + pub fn read(&self) -> LocalizationGuard { + LocalizationGuard { + active: self.active.read(), + fallback: if self.use_english_fallback { + self.fallback.map(|f| f.read()) + } else { + None + }, + } + } + + pub fn load(specifier: &str) -> Result { + let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); + let language_key = ["voxygen.i18n.", specifier].concat(); + let is_default = language_key == default_key; + Ok(Self { + active: Language::load(&language_key)?, + fallback: if is_default { + None + } else { + Language::load(&default_key).ok() + }, + use_english_fallback: false, + }) + } + + pub fn load_expect(specifier: &str) -> Self { + Self::load(specifier).expect("Can't load language files") + } + + pub fn reloaded(&mut self) -> bool { self.active.reloaded() } +} + +#[derive(Clone, Debug)] +struct LocalizationList(Vec); + +impl assets::Compound for LocalizationList { + fn load( + cache: &assets::AssetCache, + specifier: &str, + ) -> Result { + // List language directories + let mut languages = vec![]; + + let i18n_root = assets::path_of(specifier, ""); + for i18n_entry in (std::fs::read_dir(&i18n_root)?).flatten() { + if let Some(i18n_key) = i18n_entry.file_name().to_str() { + // load the root file of all the subdirectories + if let Ok(localization) = cache.load::( + &[specifier, ".", i18n_key, ".", LANG_MANIFEST_FILE].concat(), + ) { + languages.push(localization.read().metadata.clone()); + } + } + } + + Ok(LocalizationList(languages)) + } +} + +/// Load all the available languages located in the voxygen asset directory +pub fn list_localizations() -> Vec { + LocalizationList::load_expect_cloned("voxygen.i18n").0 +} + +/// Start hot reloading of i18n assets +pub fn start_hot_reloading() { assets::start_hot_reloading(); } + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis; + use std::path::Path; + + // Test that localization list is loaded (not empty) + #[test] + fn test_localization_list() { + let list = list_localizations(); + assert!(!list.is_empty()); + } + + // Test that reference language can be loaded + #[test] + fn test_localization_handle() { let _ = LocalizationHandle::load_expect(REFERENCE_LANG); } + + // Test to verify all languages that they are VALID and loadable, without + // need of git just on the local assets folder + #[test] + fn verify_all_localizations() { + // Generate paths + let i18n_asset_path = Path::new("assets/voxygen/i18n/"); + let curr_dir = std::env::current_dir().unwrap(); + let root_dir = curr_dir.parent().unwrap().parent().unwrap(); + analysis::verify_all_localizations(&root_dir, &i18n_asset_path); + } + + // Test to verify all languages and print missing and faulty localisation + #[test] + #[ignore] + fn test_all_localizations() { + // Generate paths + let i18n_asset_path = Path::new("assets/voxygen/i18n/"); + let curr_dir = std::env::current_dir().unwrap(); + let root_dir = curr_dir.parent().unwrap().parent().unwrap(); + analysis::test_all_localizations(&root_dir, &i18n_asset_path); + } +} diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs new file mode 100644 index 0000000000..9377f81aba --- /dev/null +++ b/voxygen/i18n/src/lib.rs @@ -0,0 +1,5 @@ +pub mod analysis; +mod data; + +use common_assets as assets; +pub use data::*; diff --git a/voxygen/src/i18n.rs b/voxygen/src/i18n.rs deleted file mode 100644 index cd3f823f68..0000000000 --- a/voxygen/src/i18n.rs +++ /dev/null @@ -1,940 +0,0 @@ -use common::assets::{self, AssetExt, AssetGuard, AssetHandle}; -use deunicode::deunicode; -use hashbrown::{HashMap, HashSet}; -use serde::{Deserialize, Serialize}; -use tracing::warn; - -/// The reference language, aka the more up-to-date localization data. -/// Also the default language at first startup. -pub const REFERENCE_LANG: &str = "en"; - -pub const LANG_MANIFEST_FILE: &str = "_manifest"; - -/// How a language can be described -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LanguageMetadata { - /// A human friendly language name (e.g. "English (US)") - pub language_name: String, - - /// A short text identifier for this language (e.g. "en_US") - /// - /// On the opposite of `language_name` that can change freely, - /// `language_identifier` value shall be stable in time as it - /// is used by setting components to store the language - /// selected by the user. - pub language_identifier: String, -} - -/// Store font metadata -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Font { - /// Key to retrieve the font in the asset system - pub asset_key: String, - - /// Scale ratio to resize the UI text dynamicly - pub scale_ratio: f32, -} - -impl Font { - /// Scale input size to final UI size - pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } -} - -/// Store font metadata -pub type Fonts = HashMap; - -/// Raw localization data, expect the strings to not be loaded here -/// However, metadata informations are correct -/// See `Language` for more info on each attributes -#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] -pub struct RawLocalization { - pub sub_directories: Vec, - pub string_map: HashMap, - pub vector_map: HashMap>, - pub convert_utf8_to_ascii: bool, - pub fonts: Fonts, - pub metadata: LanguageMetadata, -} - -/// Store internationalization data -#[derive(Debug, PartialEq, Serialize, Deserialize)] -struct Language { - /// A list of subdirectories to lookup for localization files - pub sub_directories: Vec, - - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub vector_map: HashMap>, - - /// Whether to convert the input text encoded in UTF-8 - /// into a ASCII version by using the `deunicode` crate. - pub convert_utf8_to_ascii: bool, - - /// Font configuration is stored here - pub fonts: Fonts, - - pub metadata: LanguageMetadata, -} - -/// Store internationalization maps -/// These structs are meant to be merged into a Language -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub struct LocalizationFragment { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub vector_map: HashMap>, -} - -impl Language { - /// Get a localized text from the given key - pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { - self.string_map.get(key).map(|s| s.as_str()) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { - self.vector_map - .get(key) - .map(|v| { - if !v.is_empty() { - Some(v[index as usize % v.len()].as_str()) - } else { - None - } - }) - .flatten() - } -} - -impl Default for Language { - fn default() -> Self { - Self { - sub_directories: Vec::default(), - string_map: HashMap::default(), - vector_map: HashMap::default(), - ..Default::default() - } - } -} - -impl From for Language { - fn from(raw: RawLocalization) -> Self { - Self { - sub_directories: raw.sub_directories, - string_map: raw.string_map, - vector_map: raw.vector_map, - convert_utf8_to_ascii: raw.convert_utf8_to_ascii, - fonts: raw.fonts, - metadata: raw.metadata, - } - } -} -impl From for LocalizationFragment { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - } - } -} - -impl assets::Asset for RawLocalization { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} -impl assets::Asset for LocalizationFragment { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} - -impl assets::Compound for Language { - fn load( - cache: &assets::AssetCache, - asset_key: &str, - ) -> Result { - let raw = cache - .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? - .cloned(); - let mut localization = Language::from(raw); - - // Walk through files in the folder, collecting localization fragment to merge - // inside the asked_localization - for localization_asset in cache.load_dir::(asset_key)?.iter() { - localization - .string_map - .extend(localization_asset.read().string_map.clone()); - localization - .vector_map - .extend(localization_asset.read().vector_map.clone()); - } - - // Use the localization's subdirectory list to load fragments from there - for sub_directory in localization.sub_directories.iter() { - for localization_asset in cache - .load_dir::(&[asset_key, ".", sub_directory].concat())? - .iter() - { - localization - .string_map - .extend(localization_asset.read().string_map.clone()); - localization - .vector_map - .extend(localization_asset.read().vector_map.clone()); - } - } - - // Update the text if UTF-8 to ASCII conversion is enabled - if localization.convert_utf8_to_ascii { - for value in localization.string_map.values_mut() { - *value = deunicode(value); - } - - for value in localization.vector_map.values_mut() { - *value = value.iter().map(|s| deunicode(s)).collect(); - } - } - localization.metadata.language_name = deunicode(&localization.metadata.language_name); - - Ok(localization) - } -} - -/// the central data structure to handle localization in veloren -// inherit Copy+Clone from AssetHandle -#[derive(Debug, PartialEq, Copy, Clone)] -pub struct LocalizationHandle { - active: AssetHandle, - fallback: Option>, - pub use_english_fallback: bool, -} - -// RAII guard returned from Localization::read(), resembles AssetGuard -pub struct LocalizationGuard { - active: AssetGuard, - fallback: Option>, -} - -// arbitrary choice to minimize changing all of veloren -pub type Localization = LocalizationGuard; - -impl LocalizationGuard { - /// Get a localized text from the given key - /// - /// First lookup is done in the active language, second in - /// the fallback (if present). - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get<'a>(&'a self, key: &'a str) -> &str { - self.active.get(key).unwrap_or_else(|| { - self.fallback - .as_ref() - .map(|f| f.get(key)) - .flatten() - .unwrap_or(key) - }) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { - self.active.get_variation(key, index).unwrap_or_else(|| { - self.fallback - .as_ref() - .map(|f| f.get_variation(key, index)) - .flatten() - .unwrap_or(key) - }) - } - - /// Return the missing keys compared to the reference language - fn list_missing_entries(&self) -> (HashSet, HashSet) { - if let Some(ref_lang) = &self.fallback { - let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); - let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); - let strings = reference_string_keys - .difference(&string_keys) - .cloned() - .collect(); - - let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); - let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); - let vectors = reference_vector_keys - .difference(&vector_keys) - .cloned() - .collect(); - - (strings, vectors) - } else { - (HashSet::default(), HashSet::default()) - } - } - - /// Log missing entries (compared to the reference language) as warnings - pub fn log_missing_entries(&self) { - let (missing_strings, missing_vectors) = self.list_missing_entries(); - for missing_key in missing_strings { - warn!( - "[{:?}] Missing string key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - for missing_key in missing_vectors { - warn!( - "[{:?}] Missing vector key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - } - - pub fn fonts(&self) -> &Fonts { &self.active.fonts } - - pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } -} - -impl LocalizationHandle { - pub fn set_english_fallback(&mut self, use_english_fallback: bool) { - self.use_english_fallback = use_english_fallback; - } - - pub fn read(&self) -> LocalizationGuard { - LocalizationGuard { - active: self.active.read(), - fallback: if self.use_english_fallback { - self.fallback.map(|f| f.read()) - } else { - None - }, - } - } - - pub fn load(specifier: &str) -> Result { - let default_key = i18n_asset_key(REFERENCE_LANG); - let is_default = specifier == default_key; - Ok(Self { - active: Language::load(specifier)?, - fallback: if is_default { - None - } else { - Language::load(&default_key).ok() - }, - use_english_fallback: false, - }) - } - - pub fn load_expect(specifier: &str) -> Self { - Self::load(specifier).expect("Can't load language files") - } - - pub fn reloaded(&mut self) -> bool { self.active.reloaded() } -} - -#[derive(Clone, Debug)] -struct LocalizationList(Vec); - -impl assets::Compound for LocalizationList { - fn load( - cache: &assets::AssetCache, - specifier: &str, - ) -> Result { - // List language directories - let mut languages = vec![]; - - let i18n_root = assets::path_of(specifier, ""); - for i18n_entry in (std::fs::read_dir(&i18n_root)?).flatten() { - if let Some(i18n_key) = i18n_entry.file_name().to_str() { - // load the root file of all the subdirectories - if let Ok(localization) = cache.load::( - &[specifier, ".", i18n_key, ".", LANG_MANIFEST_FILE].concat(), - ) { - languages.push(localization.read().metadata.clone()); - } - } - } - - Ok(LocalizationList(languages)) - } -} - -/// Load all the available languages located in the voxygen asset directory -pub fn list_localizations() -> Vec { - LocalizationList::load_expect_cloned("voxygen.i18n").0 -} - -/// Return the asset associated with the language_id -pub fn i18n_asset_key(language_id: &str) -> String { ["voxygen.i18n.", language_id].concat() } - -#[cfg(test)] -mod tests { - use super::{LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG}; - use git2::Repository; - use hashbrown::{HashMap, HashSet}; - use ron::de::{from_bytes, from_reader}; - use std::{ - fs, - path::{Path, PathBuf}, - }; - - /// List localization directories as a PathBuf vector - fn i18n_directories(i18n_dir: &Path) -> Vec { - fs::read_dir(i18n_dir) - .unwrap() - .map(|res| res.map(|e| e.path()).unwrap()) - .filter(|e| e.is_dir()) - .collect() - } - - #[derive(Debug, PartialEq)] - enum LocalizationState { - UpToDate, - NotFound, - Outdated, - Unknown, - Unused, - } - - #[derive(Debug)] - struct LocalizationEntryState { - pub key_line: Option, - pub chuck_line_range: Option<(usize, usize)>, - pub commit_id: Option, - pub state: LocalizationState, - } - - impl LocalizationEntryState { - pub fn new() -> LocalizationEntryState { - LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::Unknown, - } - } - } - - /// Returns the Git blob associated with the given reference and path - #[allow(clippy::expect_fun_call)] // TODO: Pending review in #587 - fn read_file_from_path<'a>( - repo: &'a git2::Repository, - reference: &git2::Reference, - path: &std::path::Path, - ) -> git2::Blob<'a> { - let tree = reference - .peel_to_tree() - .expect("Impossible to peel HEAD to a tree object"); - tree.get_path(path) - .expect(&format!( - "Impossible to find the file {:?} in reference {:?}", - path, - reference.name() - )) - .to_object(&repo) - .unwrap() - .peel_to_blob() - .expect("Impossible to fetch the Git object") - } - - fn correspond(line: &str, key: &str) -> bool { - let pat = { - // Get left part of split - let mut begin = line - .split(':') - .next() - .expect("split always produces value") - .trim() - .chars(); - // Remove quotes - begin.next(); - begin.next_back(); - begin.as_str() - }; - - pat == key - } - - fn generate_key_version<'a>( - repo: &'a git2::Repository, - localization: &LocalizationFragment, - path: &std::path::Path, - file_blob: &git2::Blob, - ) -> HashMap { - let mut keys: HashMap = localization - .string_map - .keys() - .map(|k| (k.to_owned(), LocalizationEntryState::new())) - .collect(); - let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); - // Find key start lines - let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); - - for (line_nb, line) in file_content.lines().enumerate() { - let mut found_key = None; - - for key in to_process.iter() { - if correspond(line, key) { - found_key = Some(key.to_owned()); - } - } - - if let Some(key) = found_key { - keys.get_mut(key).unwrap().key_line = Some(line_nb); - to_process.remove(key); - }; - } - - let mut error_check_set: Vec = vec![]; - // Find commit for each keys - repo.blame_file(path, None) - .expect("Impossible to generate the Git blame") - .iter() - .for_each(|e: git2::BlameHunk| { - for (key, state) in keys.iter_mut() { - let line = match state.key_line { - Some(l) => l, - None => { - if !error_check_set.contains(key) { - eprintln!( - "Key {} does not have a git line in it's state! Skipping key.", - key - ); - error_check_set.push(key.clone()); - } - continue; - }, - }; - - if line + 1 >= e.final_start_line() - && line + 1 < e.final_start_line() + e.lines_in_hunk() - { - state.chuck_line_range = Some(( - e.final_start_line(), - e.final_start_line() + e.lines_in_hunk(), - )); - state.commit_id = match state.commit_id { - Some(existing_commit) => { - match repo.graph_descendant_of(e.final_commit_id(), existing_commit) - { - Ok(true) => Some(e.final_commit_id()), - Ok(false) => Some(existing_commit), - Err(err) => panic!("{}", err), - } - }, - None => Some(e.final_commit_id()), - }; - } - } - }); - - keys - } - - fn complete_key_versions<'a>( - repo: &'a git2::Repository, - head_ref: &git2::Reference, - i18n_key_versions: &mut HashMap, - dir: &Path, - ) { - let root_dir = std::env::current_dir() - .map(|p| p.parent().expect("").to_owned()) - .unwrap(); - //TODO: review unwraps in this file - for i18n_file in root_dir.join(&dir).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - if file_type.is_file() { - let full_path = i18n_file.path(); - let path = full_path.strip_prefix(&root_dir).unwrap(); - println!("-> {:?}", i18n_file.file_name()); - let i18n_blob = read_file_from_path(&repo, &head_ref, &path); - let i18n: LocalizationFragment = match from_bytes(i18n_blob.content()) { - Ok(v) => v, - Err(e) => { - eprintln!( - "Could not parse {} RON file, skipping: {}", - i18n_file.path().to_string_lossy(), - e - ); - continue; - }, - }; - i18n_key_versions.extend(generate_key_version(&repo, &i18n, &path, &i18n_blob)); - } - } - } - } - - fn verify_localization_directory(directory_path: &Path) { - let root_dir = std::env::current_dir() - .map(|p| p.parent().expect("").to_owned()) - .unwrap(); - // Walk through each file in the directory - for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - // Skip folders and the manifest file (which does not contain the same struct we - // want to load) - if file_type.is_file() - && i18n_file.file_name().to_string_lossy() - != (LANG_MANIFEST_FILE.to_string() + ".ron") - { - let full_path = i18n_file.path(); - println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap()); - let f = fs::File::open(&full_path).expect("Failed opening file"); - let _: LocalizationFragment = match from_reader(f) { - Ok(v) => v, - Err(e) => { - panic!( - "Could not parse {} RON file, error: {}", - full_path.to_string_lossy(), - e - ); - }, - }; - } - } - } - } - - // Test to verify all languages that they are VALID and loadable, without - // need of git just on the local assets folder - #[test] - fn verify_all_localizations() { - // Generate paths - let i18n_asset_path = Path::new("assets/voxygen/i18n/"); - let ref_i18n_dir_path = i18n_asset_path.join(REFERENCE_LANG); - let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - let root_dir = std::env::current_dir() - .map(|p| p.parent().expect("").to_owned()) - .unwrap(); - assert!( - root_dir.join(&ref_i18n_dir_path).is_dir(), - "Reference language folder doesn't exist, something is wrong!" - ); - assert!( - root_dir.join(&ref_i18n_path).is_file(), - "Reference language manifest file doesn't exist, something is wrong!" - ); - let i18n_directories = i18n_directories(&root_dir.join(i18n_asset_path)); - // This simple check ONLY guarantees that an arbitrary minimum of translation - // files exists. It's just to notice unintentional deletion of all - // files, or modifying the paths. In case you want to delete all - // language you have to adjust this number: - assert!( - i18n_directories.len() > 5, - "have less than 5 translation folders, arbitrary minimum check failed. Maybe the i18n \ - folder is empty?" - ); - for i18n_directory in i18n_directories { - // Attempt to load the manifest file - let manifest_path = i18n_directory.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - println!( - "verifying {:?}", - manifest_path.strip_prefix(&root_dir).unwrap() - ); - let f = fs::File::open(&manifest_path).expect("Failed opening file"); - let raw_localization: RawLocalization = match from_reader(f) { - Ok(v) => v, - Err(e) => { - panic!( - "Could not parse {} RON file, error: {}", - i18n_directory.to_string_lossy(), - e - ); - }, - }; - // Walk through each files and try to load them - verify_localization_directory(&i18n_directory); - // Walk through each subdirectories and try to load files in them - for sub_directory in raw_localization.sub_directories.iter() { - let subdir_path = &i18n_directory.join(sub_directory); - verify_localization_directory(&subdir_path); - } - } - } - - // Test to verify all languages and print missing and faulty localisation - #[test] - #[ignore] - #[allow(clippy::expect_fun_call)] - fn test_all_localizations() { - // Generate paths - let i18n_asset_path = Path::new("assets/voxygen/i18n/"); - let ref_i18n_dir_path = i18n_asset_path.join(REFERENCE_LANG); - let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - let root_dir = std::env::current_dir() - .map(|p| p.parent().expect("").to_owned()) - .unwrap(); - let i18n_path = root_dir.join(i18n_asset_path); - - if !root_dir.join(&ref_i18n_dir_path).is_dir() { - panic!( - "Reference language folder not found {:?}", - &ref_i18n_dir_path - ) - } - if !root_dir.join(&ref_i18n_path).is_file() { - panic!("Reference language file not found {:?}", &ref_i18n_path) - } - - // Initialize Git objects - let repo = Repository::discover(&root_dir).expect(&format!( - "Failed to open the Git repository at {:?}", - &root_dir - )); - let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - - // Read HEAD for the reference language file - let i18n_ref_blob = read_file_from_path(&repo, &head_ref, &ref_i18n_path); - let loc: RawLocalization = from_bytes(i18n_ref_blob.content()) - .expect("Expect to parse reference i18n RON file, can't proceed without it"); - let mut i18n_references: HashMap = generate_key_version( - &repo, - &LocalizationFragment::from(loc.clone()), - &ref_i18n_path, - &i18n_ref_blob, - ); - - // read HEAD for the fragment files - complete_key_versions(&repo, &head_ref, &mut i18n_references, &ref_i18n_dir_path); - // read HEAD for the subfolders - for sub_directory in loc.sub_directories.iter() { - let subdir_path = &ref_i18n_dir_path.join(sub_directory); - complete_key_versions(&repo, &head_ref, &mut i18n_references, &subdir_path); - } - - // Compare to other reference files - let i18n_directories = i18n_directories(&i18n_path); - let mut i18n_entry_counts: HashMap = HashMap::new(); - for file in &i18n_directories { - let reldir = file.strip_prefix(&root_dir).unwrap(); - let relfile = reldir.join(&(LANG_MANIFEST_FILE.to_string() + ".ron")); - if relfile == ref_i18n_path { - continue; - } - println!("\n-----------------------------------"); - println!("{:?}", relfile); - println!("-----------------------------------"); - - // Find the localization entry state - let current_blob = read_file_from_path(&repo, &head_ref, &relfile); - let current_loc: RawLocalization = match from_bytes(current_blob.content()) { - Ok(v) => v, - Err(e) => { - eprintln!( - "Could not parse {} RON file, skipping: {}", - relfile.to_string_lossy(), - e - ); - continue; - }, - }; - let mut current_i18n = generate_key_version( - &repo, - &LocalizationFragment::from(current_loc.clone()), - &relfile, - ¤t_blob, - ); - // read HEAD for the fragment files - complete_key_versions(&repo, &head_ref, &mut current_i18n, &reldir); - // read HEAD for the subfolders - for sub_directory in current_loc.sub_directories.iter() { - let subdir_path = &reldir.join(sub_directory); - complete_key_versions(&repo, &head_ref, &mut current_i18n, &subdir_path); - } - - for (ref_key, ref_state) in i18n_references.iter() { - match current_i18n.get_mut(ref_key) { - Some(state) => { - let commit_id = match state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in i18n file {} is missing! Skipping key.", - ref_key, - relfile.to_string_lossy() - ); - continue; - }, - }; - let ref_commit_id = match ref_state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in reference i18n file is missing! \ - Skipping key.", - ref_key - ); - continue; - }, - }; - if commit_id != ref_commit_id - && !repo - .graph_descendant_of(commit_id, ref_commit_id) - .unwrap_or(false) - { - state.state = LocalizationState::Outdated; - } else { - state.state = LocalizationState::UpToDate; - } - }, - None => { - current_i18n.insert(ref_key.to_owned(), LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::NotFound, - }); - }, - } - } - - let ref_keys: HashSet<&String> = i18n_references.keys().collect(); - for (_, state) in current_i18n - .iter_mut() - .filter(|&(k, _)| !ref_keys.contains(k)) - { - state.state = LocalizationState::Unused; - } - - // Display - println!( - "\n{:10} | {:60}| {:40} | {:40}\n", - "State", - "Key name", - relfile.to_str().unwrap(), - ref_i18n_path.to_str().unwrap() - ); - - let mut sorted_keys: Vec<&String> = current_i18n.keys().collect(); - sorted_keys.sort(); - - let current_i18n_entry_count = current_i18n.len(); - let mut uptodate_entries = 0; - let mut outdated_entries = 0; - let mut unused_entries = 0; - let mut notfound_entries = 0; - let mut unknown_entries = 0; - - for key in sorted_keys { - let state = current_i18n.get(key).unwrap(); - if state.state != LocalizationState::UpToDate { - match state.state { - LocalizationState::Outdated => outdated_entries += 1, - LocalizationState::NotFound => notfound_entries += 1, - LocalizationState::Unknown => unknown_entries += 1, - LocalizationState::Unused => unused_entries += 1, - LocalizationState::UpToDate => unreachable!(), - }; - - println!( - "[{:9}] | {:60}| {:40} | {:40}", - format!("{:?}", state.state), - key, - state - .commit_id - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_string()), - i18n_references - .get(key) - .map(|s| s.commit_id) - .flatten() - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_string()), - ); - } else { - uptodate_entries += 1; - } - } - - println!( - "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", - uptodate_entries, - outdated_entries, - unused_entries, - notfound_entries, - unknown_entries - ); - - // Calculate key count that actually matter for the status of the translation - // Unused entries don't break the game - let real_entry_count = current_i18n_entry_count - unused_entries; - let uptodate_percent = (uptodate_entries as f32 / real_entry_count as f32) * 100_f32; - let outdated_percent = (outdated_entries as f32 / real_entry_count as f32) * 100_f32; - let untranslated_percent = - ((notfound_entries + unknown_entries) as f32 / real_entry_count as f32) * 100_f32; - - println!( - "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", - uptodate_percent, outdated_percent, untranslated_percent, - ); - - i18n_entry_counts.insert( - file.clone(), - ( - uptodate_entries, - outdated_entries, - notfound_entries + unknown_entries, - real_entry_count, - ), - ); - } - - let mut overall_uptodate_entry_count = 0; - let mut overall_outdated_entry_count = 0; - let mut overall_untranslated_entry_count = 0; - let mut overall_real_entry_count = 0; - - println!("-----------------------------------------------------------------------------"); - println!("Overall Translation Status"); - println!("-----------------------------------------------------------------------------"); - println!( - "{:12}| {:8} | {:8} | {:8}", - "", "up-to-date", "outdated", "untranslated" - ); - - for (path, (uptodate, outdated, untranslated, real)) in i18n_entry_counts { - overall_uptodate_entry_count += uptodate; - overall_outdated_entry_count += outdated; - overall_untranslated_entry_count += untranslated; - overall_real_entry_count += real; - - println!( - "{:12}|{:8} |{:6} |{:8}", - path.file_name().unwrap().to_string_lossy(), - uptodate, - outdated, - untranslated - ); - } - - println!( - "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", - (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - ); - println!("-----------------------------------------------------------------------------\n"); - } -} diff --git a/voxygen/src/lib.rs b/voxygen/src/lib.rs index 81df2ba090..aa2af1d956 100644 --- a/voxygen/src/lib.rs +++ b/voxygen/src/lib.rs @@ -19,7 +19,6 @@ pub mod controller; mod ecs; pub mod error; pub mod hud; -pub mod i18n; pub mod key_state; pub mod menu; pub mod mesh; @@ -35,12 +34,12 @@ pub mod window; // Reexports pub use crate::error::Error; +pub use i18n; #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; use crate::{ audio::AudioFrontend, - i18n::LocalizationHandle, profile::Profile, render::Renderer, settings::Settings, @@ -48,6 +47,7 @@ use crate::{ }; use common::clock::Clock; use common_base::span; +use i18n::LocalizationHandle; /// A type used to store state that is shared between all play states. pub struct GlobalState { diff --git a/voxygen/src/main.rs b/voxygen/src/main.rs index 63ba0d7c48..89ad6e7b9d 100644 --- a/voxygen/src/main.rs +++ b/voxygen/src/main.rs @@ -5,7 +5,7 @@ use veloren_voxygen::{ audio::AudioFrontend, - i18n::{self, i18n_asset_key, LocalizationHandle}, + i18n::{self, LocalizationHandle}, profile::Profile, run, scene::terrain::SpriteRenderContext, @@ -144,6 +144,7 @@ fn main() { })); assets::start_hot_reloading(); + i18n::start_hot_reloading(); // Initialise watcher for animation hotreloading #[cfg(feature = "hot-anim")] @@ -163,8 +164,8 @@ fn main() { // Load the profile. let profile = Profile::load(); - let mut i18n = LocalizationHandle::load(&i18n_asset_key(&settings.language.selected_language)) - .unwrap_or_else(|error| { + let mut i18n = + LocalizationHandle::load(&settings.language.selected_language).unwrap_or_else(|error| { let selected_language = &settings.language.selected_language; warn!( ?error, @@ -172,7 +173,7 @@ fn main() { "Impossible to load language: change to the default language (English) instead.", ); settings.language.selected_language = i18n::REFERENCE_LANG.to_owned(); - LocalizationHandle::load_expect(&i18n_asset_key(&settings.language.selected_language)) + LocalizationHandle::load_expect(&settings.language.selected_language) }); i18n.read().log_missing_entries(); i18n.set_english_fallback(settings.language.use_english_fallback); diff --git a/voxygen/src/menu/main/mod.rs b/voxygen/src/menu/main/mod.rs index 642d902871..dac729c725 100644 --- a/voxygen/src/menu/main/mod.rs +++ b/voxygen/src/menu/main/mod.rs @@ -5,7 +5,7 @@ use super::char_selection::CharSelectionState; #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; use crate::{ - i18n::{i18n_asset_key, Localization, LocalizationHandle}, + i18n::{Localization, LocalizationHandle}, render::Renderer, settings::Settings, window::Event, @@ -295,9 +295,9 @@ impl PlayState for MainMenuState { MainMenuEvent::ChangeLanguage(new_language) => { global_state.settings.language.selected_language = new_language.language_identifier; - global_state.i18n = LocalizationHandle::load_expect(&i18n_asset_key( + global_state.i18n = LocalizationHandle::load_expect( &global_state.settings.language.selected_language, - )); + ); global_state.i18n.read().log_missing_entries(); global_state .i18n diff --git a/voxygen/src/session/settings_change.rs b/voxygen/src/session/settings_change.rs index c1e674bbcd..9426e798ae 100644 --- a/voxygen/src/session/settings_change.rs +++ b/voxygen/src/session/settings_change.rs @@ -5,7 +5,7 @@ use crate::{ BarNumbers, BuffPosition, CrosshairType, Intro, PressBehavior, ScaleChange, ShortcutNumbers, XpBar, }, - i18n::{i18n_asset_key, LanguageMetadata, LocalizationHandle}, + i18n::{LanguageMetadata, LocalizationHandle}, render::RenderMode, settings::{ AudioSettings, ControlSettings, Fps, GamepadSettings, GameplaySettings, GraphicsSettings, @@ -482,9 +482,8 @@ impl SettingsChange { SettingsChange::Language(language_change) => match language_change { Language::ChangeLanguage(new_language) => { settings.language.selected_language = new_language.language_identifier; - global_state.i18n = LocalizationHandle::load_expect(&i18n_asset_key( - &settings.language.selected_language, - )); + global_state.i18n = + LocalizationHandle::load_expect(&settings.language.selected_language); global_state.i18n.read().log_missing_entries(); global_state .i18n