diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index d0328e63a7..89841df7d1 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -1,7 +1,7 @@ use ron::de::from_bytes; use std::path::{Path, PathBuf}; -use crate::data::{ +use crate::raw::{ i18n_directories, LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG, }; use hashbrown::{HashMap, HashSet}; @@ -27,6 +27,7 @@ struct LocalizationStats { #[derive(Default)] struct LocalizationAnalysis { + uptodate: Vec<(String, Option)>, notfound: Vec<(String, Option)>, unused: Vec<(String, Option)>, outdated: Vec<(String, Option)>, @@ -39,11 +40,11 @@ impl LocalizationAnalysis { state: LocalizationState, ) -> Option<&mut Vec<(String, Option)>> { match state { + LocalizationState::UpToDate => Some(&mut self.uptodate), LocalizationState::NotFound => Some(&mut self.notfound), LocalizationState::Unused => Some(&mut self.unused), LocalizationState::Outdated => Some(&mut self.outdated), LocalizationState::Unknown => Some(&mut self.unknown), - _ => None, } } @@ -53,9 +54,7 @@ impl LocalizationAnalysis { be_verbose: bool, ref_i18n_map: &HashMap, ) { - let entries = self - .get_mut(state) - .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); + let entries = self.unwrap_entries(state); if entries.is_empty() { return; } @@ -63,9 +62,7 @@ impl LocalizationAnalysis { entries.sort(); for (key, commit_id) in entries { if be_verbose { - let our_commit = commit_id - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()); + let our_commit = LocalizationAnalysis::create_our_commit(commit_id); let ref_commit = ref_i18n_map .get(key) .and_then(|s| s.commit_id) @@ -77,6 +74,32 @@ impl LocalizationAnalysis { } } } + + //TODO: Add which file each faulty translation is in + fn csv(&mut self, state: LocalizationState) { + let entries = self.unwrap_entries(state); + for (key, commit_id) in entries { + let our_commit = LocalizationAnalysis::create_our_commit(commit_id); + println!( + "{},{},{},{:?},{}", + "sv", "_manifest.yml", key, state, our_commit + ); + } + } + + fn unwrap_entries( + &mut self, + state: LocalizationState, + ) -> &mut Vec<(String, Option)> { + self.get_mut(state) + .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)) + } + + fn create_our_commit(commit_id: &mut Option) -> String { + commit_id + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()) + } } #[derive(Copy, Clone, Debug)] @@ -329,14 +352,7 @@ fn test_localization_directory( let mut state_map = LocalizationAnalysis::default(); let result = gather_results(current_i18n, &mut state_map); - print_translation_stats( - i18n_references, - &result, - &mut state_map, - be_verbose, - relfile, - ref_manifest, - ); + print_csv_file(&mut state_map, relfile); Some(result) } @@ -490,6 +506,16 @@ fn print_translation_stats( ); } +fn print_csv_file(state_map: &mut LocalizationAnalysis, relfile: PathBuf) { + println!("country_code,file_name,translation_code,status,git_commit"); + + state_map.csv(LocalizationState::UpToDate); + state_map.csv(LocalizationState::NotFound); + state_map.csv(LocalizationState::Unused); + state_map.csv(LocalizationState::Outdated); + state_map.csv(LocalizationState::Unknown); +} + /// Test one language /// `code` - name of the directory in assets (de_DE for example) /// `root_dir` - absolute path to main repo @@ -548,7 +574,12 @@ pub fn test_specific_localization( /// `assets_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') /// csv_enabled - generate csv files in target folder -pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool, csv_enabled: bool) { +pub fn test_all_localizations( + root_dir: &Path, + assets_path: &Path, + be_verbose: bool, + csv_enabled: bool, +) { let ref_lang_dir = assets_path.join(REFERENCE_LANG); let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index 6af88157e7..f3bdf0a490 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -50,7 +50,12 @@ fn main() { ); } if matches.is_present("test") { - analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose"), csv_enabled); + analysis::test_all_localizations( + &root, + &asset_path, + matches.is_present("verbose"), + csv_enabled, + ); } if matches.is_present("verify") { verification::verify_all_localizations(&root, &asset_path); diff --git a/voxygen/i18n/src/data.rs b/voxygen/i18n/src/data.rs deleted file mode 100644 index 2240134fe2..0000000000 --- a/voxygen/i18n/src/data.rs +++ /dev/null @@ -1,437 +0,0 @@ -use crate::assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; -use deunicode::deunicode; -use hashbrown::{HashMap, HashSet}; -use serde::{Deserialize, Serialize}; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; -use tracing::warn; - -/// The reference language, aka the more up-to-date localization data. -/// Also the default language at first startup. -pub const REFERENCE_LANG: &str = "en"; - -pub const LANG_MANIFEST_FILE: &str = "_manifest"; - -/// How a language can be described -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LanguageMetadata { - /// A human friendly language name (e.g. "English (US)") - pub language_name: String, - - /// A short text identifier for this language (e.g. "en_US") - /// - /// On the opposite of `language_name` that can change freely, - /// `language_identifier` value shall be stable in time as it - /// is used by setting components to store the language - /// selected by the user. - pub language_identifier: String, -} - -/// Store font metadata -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Font { - /// Key to retrieve the font in the asset system - pub asset_key: String, - - /// Scale ratio to resize the UI text dynamicly - scale_ratio: f32, -} - -impl Font { - /// Scale input size to final UI size - pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } -} - -/// Store font metadata -pub type Fonts = HashMap; - -/// Raw localization data, expect the strings to not be loaded here -/// However, metadata informations are correct -/// See `Language` for more info on each attributes -#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] -pub(crate) struct RawLocalization { - pub(crate) convert_utf8_to_ascii: bool, - pub(crate) fonts: Fonts, - pub(crate) metadata: LanguageMetadata, - pub(crate) string_map: HashMap, - pub(crate) vector_map: HashMap>, -} - -/// Store internationalization data -#[derive(Debug, PartialEq, Serialize, Deserialize)] -struct Language { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, - - /// Whether to convert the input text encoded in UTF-8 - /// into a ASCII version by using the `deunicode` crate. - pub(crate) convert_utf8_to_ascii: bool, - - /// Font configuration is stored here - pub(crate) fonts: Fonts, - - pub(crate) metadata: LanguageMetadata, -} - -/// Store internationalization maps -/// These structs are meant to be merged into a Language -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub(crate) struct LocalizationFragment { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, -} - -impl Language { - /// Get a localized text from the given key - pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { - self.string_map.get(key).map(String::as_str) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { - self.vector_map.get(key).and_then(|v| { - if v.is_empty() { - None - } else { - Some(v[index as usize % v.len()].as_str()) - } - }) - } -} - -impl Default for Language { - fn default() -> Self { - Self { - string_map: HashMap::default(), - vector_map: HashMap::default(), - ..Default::default() - } - } -} - -impl From for Language { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - convert_utf8_to_ascii: raw.convert_utf8_to_ascii, - fonts: raw.fonts, - metadata: raw.metadata, - } - } -} -impl From for LocalizationFragment { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - } - } -} - -impl assets::Asset for RawLocalization { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} -impl assets::Asset for LocalizationFragment { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} - -impl assets::Compound for Language { - fn load( - cache: &assets::AssetCache, - asset_key: &str, - ) -> Result { - let raw = cache - .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? - .cloned(); - let mut localization = Language::from(raw); - - // Walk through files in the folder, collecting localization fragment to merge - // inside the asked_localization - for localization_asset in cache - .load_dir::(asset_key, true)? - .iter() - { - localization - .string_map - .extend(localization_asset.read().string_map.clone()); - localization - .vector_map - .extend(localization_asset.read().vector_map.clone()); - } - - // Update the text if UTF-8 to ASCII conversion is enabled - if localization.convert_utf8_to_ascii { - for value in localization.string_map.values_mut() { - *value = deunicode(value); - } - - for value in localization.vector_map.values_mut() { - *value = value.iter().map(|s| deunicode(s)).collect(); - } - } - localization.metadata.language_name = deunicode(&localization.metadata.language_name); - - Ok(localization) - } -} - -/// the central data structure to handle localization in veloren -// inherit Copy+Clone from AssetHandle -#[derive(Debug, PartialEq, Copy, Clone)] -pub struct LocalizationHandle { - active: AssetHandle, - fallback: Option>, - pub use_english_fallback: bool, -} - -// RAII guard returned from Localization::read(), resembles AssetGuard -pub struct LocalizationGuard { - active: AssetGuard, - fallback: Option>, -} - -// arbitrary choice to minimize changing all of veloren -pub type Localization = LocalizationGuard; - -impl LocalizationGuard { - /// Get a localized text from the given key - /// - /// First lookup is done in the active language, second in - /// the fallback (if present). - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get<'a>(&'a self, key: &'a str) -> &str { - self.active.get(key).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get(key)) - .unwrap_or(key) - }) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { - self.active.get_variation(key, index).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get_variation(key, index)) - .unwrap_or(key) - }) - } - - /// Return the missing keys compared to the reference language - fn list_missing_entries(&self) -> (HashSet, HashSet) { - if let Some(ref_lang) = &self.fallback { - let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); - let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); - let strings = reference_string_keys - .difference(&string_keys) - .cloned() - .collect(); - - let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); - let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); - let vectors = reference_vector_keys - .difference(&vector_keys) - .cloned() - .collect(); - - (strings, vectors) - } else { - (HashSet::default(), HashSet::default()) - } - } - - /// Log missing entries (compared to the reference language) as warnings - pub fn log_missing_entries(&self) { - let (missing_strings, missing_vectors) = self.list_missing_entries(); - for missing_key in missing_strings { - warn!( - "[{:?}] Missing string key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - for missing_key in missing_vectors { - warn!( - "[{:?}] Missing vector key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - } - - pub fn fonts(&self) -> &Fonts { &self.active.fonts } - - pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } -} - -impl LocalizationHandle { - pub fn set_english_fallback(&mut self, use_english_fallback: bool) { - self.use_english_fallback = use_english_fallback; - } - - pub fn read(&self) -> LocalizationGuard { - LocalizationGuard { - active: self.active.read(), - fallback: if self.use_english_fallback { - self.fallback.map(|f| f.read()) - } else { - None - }, - } - } - - pub fn load(specifier: &str) -> Result { - let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); - let language_key = ["voxygen.i18n.", specifier].concat(); - let is_default = language_key == default_key; - Ok(Self { - active: Language::load(&language_key)?, - fallback: if is_default { - None - } else { - Language::load(&default_key).ok() - }, - use_english_fallback: false, - }) - } - - pub fn load_expect(specifier: &str) -> Self { - Self::load(specifier).expect("Can't load language files") - } - - pub fn reloaded(&mut self) -> bool { self.active.reloaded() } -} - -struct FindManifests; - -impl assets::Compound for FindManifests { - fn load(_: &assets::AssetCache, _: &str) -> Result { - Ok(Self) - } -} - -impl assets::DirLoadable for FindManifests { - fn select_ids( - source: &S, - specifier: &str, - ) -> io::Result> { - let mut specifiers = Vec::new(); - - source.read_dir(specifier, &mut |entry| { - if let DirEntry::Directory(spec) = entry { - let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); - if source.exists(DirEntry::File(&manifest_spec, "ron")) { - specifiers.push(manifest_spec.into()); - } - } - })?; - - Ok(specifiers) - } -} - -#[derive(Clone, Debug)] -struct LocalizationList(Vec); - -impl assets::Compound for LocalizationList { - fn load( - cache: &assets::AssetCache, - specifier: &str, - ) -> Result { - // List language directories - let languages = assets::load_dir::(specifier, false) - .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) - .ids() - .filter_map(|spec| cache.load::(spec).ok()) - .map(|localization| localization.read().metadata.clone()) - .collect(); - - Ok(LocalizationList(languages)) - } -} - -/// Load all the available languages located in the voxygen asset directory -pub fn list_localizations() -> Vec { - LocalizationList::load_expect_cloned("voxygen.i18n").0 -} - -/// List localization directories as a `PathBuf` vector -pub fn i18n_directories(i18n_dir: &Path) -> Vec { - fs::read_dir(i18n_dir) - .unwrap() - .map(|res| res.map(|e| e.path()).unwrap()) - .filter(|e| e.is_dir()) - .collect() -} - -#[cfg(test)] -mod tests { - use super::assets; - // Test that localization list is loaded (not empty) - #[test] - fn test_localization_list() { - let list = super::list_localizations(); - assert!(!list.is_empty()); - } - - // Test that reference language can be loaded - #[test] - fn test_localization_handle() { - let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); - } - - // Test to verify all languages that they are VALID and loadable, without - // need of git just on the local assets folder - #[test] - fn verify_all_localizations() { - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::verification::verify_all_localizations(&root_dir, i18n_asset_path); - } - - // Test to verify all languages and print missing and faulty localisation - #[test] - #[ignore] - fn test_all_localizations() { - // Options - let be_verbose = true; - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&root_dir, i18n_asset_path, be_verbose); - } -} diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 1d8462e15d..d0b9f85e8c 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,7 +1,365 @@ #[cfg(any(feature = "bin", test))] pub mod analysis; -mod data; +pub mod raw; pub mod verification; -use common_assets as assets; -pub use data::*; +use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; +use hashbrown::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; +use tracing::warn; +use raw::{RawManifest, RawFragment, RawLanguage}; + +/// The reference language, aka the more up-to-date localization data. +/// Also the default language at first startup. +pub const REFERENCE_LANG: &str = "en"; + +pub const LANG_MANIFEST_FILE: &str = "_manifest"; + +pub(crate) const LANG_EXTENSION: &str = "ron"; + +/// How a language can be described +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LanguageMetadata { + /// A human friendly language name (e.g. "English (US)") + pub language_name: String, + + /// A short text identifier for this language (e.g. "en_US") + /// + /// On the opposite of `language_name` that can change freely, + /// `language_identifier` value shall be stable in time as it + /// is used by setting components to store the language + /// selected by the user. + pub language_identifier: String, +} + +/// Store font metadata +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Font { + /// Key to retrieve the font in the asset system + pub asset_key: String, + + /// Scale ratio to resize the UI text dynamically + scale_ratio: f32, +} + +impl Font { + /// Scale input size to final UI size + pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } +} + +/// Store font metadata +pub type Fonts = HashMap; + +/// Store internationalization data +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Language { + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + pub(crate) string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + pub(crate) vector_map: HashMap>, + + /// Whether to convert the input text encoded in UTF-8 + /// into a ASCII version by using the `deunicode` crate. + pub(crate) convert_utf8_to_ascii: bool, + + /// Font configuration is stored here + pub(crate) fonts: Fonts, + + pub(crate) metadata: LanguageMetadata, +} + +impl Language { + /// Get a localized text from the given key + pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { + self.string_map.get(key).map(String::as_str) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { + self.vector_map.get(key).and_then(|v| { + if v.is_empty() { + None + } else { + Some(v[index as usize % v.len()].as_str()) + } + }) + } +} + +impl common_assets::Compound for Language { + fn load( + cache: &common_assets::AssetCache, + asset_key: &str, + ) -> Result { + let manifest = cache + .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? + .cloned(); + + // Walk through files in the folder, collecting localization fragment to merge + // inside the asked_localization + let mut fragments = HashMap::new(); + for fragment_asset in cache + .load_dir::(asset_key, true)? + .iter() + { + let read = fragment_asset.read(); + fragments.insert(PathBuf::from(fragment_asset.id()), read.clone()); + } + + Ok(Language::from(RawLanguage{ + manifest, + fragments, + })) + } +} + +/// the central data structure to handle localization in veloren +// inherit Copy+Clone from AssetHandle +#[derive(Debug, PartialEq, Copy, Clone)] +pub struct LocalizationHandle { + active: AssetHandle, + fallback: Option>, + pub use_english_fallback: bool, +} + +// RAII guard returned from Localization::read(), resembles AssetGuard +pub struct LocalizationGuard { + active: AssetGuard, + fallback: Option>, +} + +// arbitrary choice to minimize changing all of veloren +pub type Localization = LocalizationGuard; + +impl LocalizationGuard { + /// Get a localized text from the given key + /// + /// First lookup is done in the active language, second in + /// the fallback (if present). + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get<'a>(&'a self, key: &'a str) -> &str { + self.active.get(key).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get(key)) + .unwrap_or(key) + }) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { + self.active.get_variation(key, index).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get_variation(key, index)) + .unwrap_or(key) + }) + } + + /// Return the missing keys compared to the reference language + fn list_missing_entries(&self) -> (HashSet, HashSet) { + if let Some(ref_lang) = &self.fallback { + let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); + let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); + let strings = reference_string_keys + .difference(&string_keys) + .cloned() + .collect(); + + let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); + let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); + let vectors = reference_vector_keys + .difference(&vector_keys) + .cloned() + .collect(); + + (strings, vectors) + } else { + (HashSet::default(), HashSet::default()) + } + } + + /// Log missing entries (compared to the reference language) as warnings + pub fn log_missing_entries(&self) { + let (missing_strings, missing_vectors) = self.list_missing_entries(); + for missing_key in missing_strings { + warn!( + "[{:?}] Missing string key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + for missing_key in missing_vectors { + warn!( + "[{:?}] Missing vector key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + } + + pub fn fonts(&self) -> &Fonts { &self.active.fonts } + + pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } +} + +impl LocalizationHandle { + pub fn set_english_fallback(&mut self, use_english_fallback: bool) { + self.use_english_fallback = use_english_fallback; + } + + pub fn read(&self) -> LocalizationGuard { + LocalizationGuard { + active: self.active.read(), + fallback: if self.use_english_fallback { + self.fallback.map(|f| f.read()) + } else { + None + }, + } + } + + pub fn load(specifier: &str) -> Result { + let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); + let language_key = ["voxygen.i18n.", specifier].concat(); + let is_default = language_key == default_key; + Ok(Self { + active: Language::load(&language_key)?, + fallback: if is_default { + None + } else { + Language::load(&default_key).ok() + }, + use_english_fallback: false, + }) + } + + pub fn load_expect(specifier: &str) -> Self { + Self::load(specifier).expect("Can't load language files") + } + + pub fn reloaded(&mut self) -> bool { self.active.reloaded() } +} + +struct FindManifests; + +impl common_assets::Compound for FindManifests { + fn load(_: &common_assets::AssetCache, _: &str) -> Result { + Ok(Self) + } +} + +impl common_assets::DirLoadable for FindManifests { + fn select_ids( + source: &S, + specifier: &str, + ) -> io::Result> { + let mut specifiers = Vec::new(); + + source.read_dir(specifier, &mut |entry| { + if let DirEntry::Directory(spec) = entry { + let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); + if source.exists(DirEntry::File(&manifest_spec, LANG_EXTENSION)) { + specifiers.push(manifest_spec.into()); + } + } + })?; + + Ok(specifiers) + } +} + +#[derive(Clone, Debug)] +struct LocalizationList(Vec); + +impl common_assets::Compound for LocalizationList { + fn load( + cache: &common_assets::AssetCache, + specifier: &str, + ) -> Result { + // List language directories + let languages = common_assets::load_dir::(specifier, false) + .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) + .ids() + .filter_map(|spec| cache.load::(spec).ok()) + .map(|localization| localization.read().metadata.clone()) + .collect(); + + Ok(LocalizationList(languages)) + } +} + +/// Load all the available languages located in the voxygen asset directory +pub fn list_localizations() -> Vec { + LocalizationList::load_expect_cloned("voxygen.i18n").0 +} + +/// List localization directories as a `PathBuf` vector +pub fn i18n_directories(i18n_dir: &Path) -> Vec { + fs::read_dir(i18n_dir) + .unwrap() + .map(|res| res.map(|e| e.path()).unwrap()) + .filter(|e| e.is_dir()) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use common_assets; + + // Test that localization list is loaded (not empty) + #[test] + fn test_localization_list() { + let list = super::list_localizations(); + assert!(!list.is_empty()); + } + + // Test that reference language can be loaded + #[test] + fn test_localization_handle() { + let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); + } + + // Test to verify all languages that they are VALID and loadable, without + // need of git just on the local assets folder + #[test] + fn verify_all_localizations() { + // Generate paths + let i18n_root_path = Path::new("assets/voxygen/i18n/"); + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::verification::verify_all_localizations(&root_dir, i18n_root_path); + } + + // Test to verify all languages and print missing and faulty localisation + #[test] + #[ignore] + fn test_all_localizations() { + // Options + let be_verbose = true; + // Generate paths + let i18n_root_path = Path::new("assets/voxygen/i18n/"); + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::analysis::test_all_localizations(&root_dir, i18n_root_path, be_verbose); + } +} diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs new file mode 100644 index 0000000000..e04d64e706 --- /dev/null +++ b/voxygen/i18n/src/raw.rs @@ -0,0 +1,136 @@ +//! handle the loading of a `Language` +use hashbrown::hash_map::HashMap; +use std::path::{Path, PathBuf}; +use serde::{Deserialize, Serialize}; +use std::fs; +use ron::de::from_reader; +use deunicode::deunicode; +use crate::{Fonts, LanguageMetadata, LANG_MANIFEST_FILE, LANG_EXTENSION}; +use crate::Language; + +/// Raw localization metadata from LANG_MANIFEST_FILE file +/// See `Language` for more info on each attributes +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawManifest { + pub(crate) convert_utf8_to_ascii: bool, + pub(crate) fonts: Fonts, + pub(crate) metadata: LanguageMetadata, +} + +/// Raw localization data from one specific file +/// These structs are meant to be merged into a Language +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawFragment { + pub(crate) string_map: HashMap, + pub(crate) vector_map: HashMap>, +} + +pub(crate) struct RawLanguage { + pub(crate) manifest: RawManifest, + pub(crate) fragments: HashMap, +} + +#[derive(Debug)] +pub(crate) enum RawError { + RonError(ron::Error), +} + +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` folders +pub(crate) fn load_manifest(i18n_root_path: &Path, language_identifier: &str) -> Result { + let manifest_file = i18n_root_path.join(language_identifier).join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); + println!("file , {:?}", manifest_file); + let f = fs::File::open(&manifest_file)?; + Ok(from_reader(f).map_err(RawError::RonError)?) +} + +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` files +pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result { + // Walk through each file in the directory + let mut fragments = HashMap::new(); + let language_identifier = &manifest.metadata.language_identifier; + let language_dir = i18n_root_path.join(language_identifier); + for fragment_file in language_dir.read_dir().unwrap().flatten() { + let file_type = fragment_file.file_type()?; + if file_type.is_dir() { + // TODO: recursive + continue; + } + if file_type.is_file() { + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(&i18n_root_path).unwrap(); + let f = fs::File::open(&full_path)?; + let fragment = from_reader(f).map_err(RawError::RonError)?; + fragments.insert(relative_path.to_path_buf(), fragment); + } + } + Ok(RawLanguage{ + manifest, + fragments, + }) +} + +impl From for Language { + fn from(raw: RawLanguage) -> Self { + + let mut string_map = HashMap::new(); + let mut vector_map = HashMap::new(); + + for (_, fragment) in raw.fragments { + string_map.extend(fragment.string_map); + vector_map.extend(fragment.vector_map); + } + + let convert_utf8_to_ascii = raw.manifest.convert_utf8_to_ascii; + + // Update the text if UTF-8 to ASCII conversion is enabled + if convert_utf8_to_ascii { + for value in string_map.values_mut() { + *value = deunicode(value); + } + + for value in vector_map.values_mut() { + *value = value.iter().map(|s| deunicode(s)).collect(); + } + } + let mut metadata = raw.manifest.metadata; + metadata.language_name = deunicode(&metadata.language_name); + + Self { + string_map, + vector_map, + convert_utf8_to_ascii, + fonts: raw.manifest.fonts, + metadata: metadata, + } + } +} + +impl core::fmt::Display for RawError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + RawError::RonError(e) => write!(f, "{}", e), + } + } +} + +impl std::error::Error for RawError {} + + +impl From for common_assets::Error { + fn from(e: RawError) -> Self { + Self::Conversion(Box::new(e)) + } +} + + +impl common_assets::Asset for RawManifest { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} + +impl common_assets::Asset for RawFragment { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} \ No newline at end of file diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index ad7897c386..9a62974abc 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -1,34 +1,7 @@ -use ron::de::from_reader; -use std::{fs, path::Path}; +use std::{path::Path}; -use crate::data::{i18n_directories, LocalizationFragment, LANG_MANIFEST_FILE, REFERENCE_LANG}; - -fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { - // Walk through each file in the directory - for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - // Skip folders and the manifest file (which does not contain the same struct we - // want to load) - if file_type.is_file() { - let full_path = i18n_file.path(); - println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap()); - let f = fs::File::open(&full_path).expect("Failed opening file"); - let _loc: LocalizationFragment = match from_reader(f) { - Ok(v) => v, - Err(e) => { - panic!( - "Could not parse {} RON file, error: {}", - full_path.to_string_lossy(), - e - ); - }, - }; - } else if file_type.is_dir() { - verify_localization_directory(root_dir, &i18n_file.path()); - } - } - } -} +use crate::{i18n_directories, LANG_MANIFEST_FILE, REFERENCE_LANG}; +use crate::raw; /// Test to verify all languages that they are VALID and loadable, without /// need of git just on the local assets folder @@ -36,17 +9,18 @@ fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { /// `asset_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { - let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG); - let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + let i18n_root_path = root_dir.join(asset_path); + let ref_i18n_path = i18n_root_path.join(REFERENCE_LANG); + let ref_i18n_manifest_path = ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION); assert!( - root_dir.join(&ref_i18n_dir_path).is_dir(), + root_dir.join(&ref_i18n_path).is_dir(), "Reference language folder doesn't exist, something is wrong!" ); assert!( - root_dir.join(&ref_i18n_path).is_file(), + root_dir.join(&ref_i18n_manifest_path).is_file(), "Reference language manifest file doesn't exist, something is wrong!" ); - let i18n_directories = i18n_directories(&root_dir.join(asset_path)); + let i18n_directories = i18n_directories(&i18n_root_path); // This simple check ONLY guarantees that an arbitrary minimum of translation // files exists. It's just to notice unintentional deletion of all // files, or modifying the paths. In case you want to delete all @@ -57,11 +31,19 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { folder is empty?" ); for i18n_directory in i18n_directories { + let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().as_os_str().to_str().unwrap(); + let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().as_os_str().to_str().unwrap(); println!( "verifying {:?}", - i18n_directory.strip_prefix(&root_dir).unwrap() + display_language_identifier ); // Walk through each files and try to load them - verify_localization_directory(root_dir, &i18n_directory); + verify_localization_directory(root_dir, &asset_path, language_identifier); } } + +fn verify_localization_directory(root_dir: &Path, asset_path: &Path, language_identifier: &str) { + let i18n_path = root_dir.join(asset_path); + let manifest = raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file"); + raw::load_raw_language(&i18n_path, manifest).expect("error accessing fragment file"); +} \ No newline at end of file