From c9c32eea65fc723d8c90ef54dec59b1b8e6bceb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Fri, 23 Jul 2021 12:25:59 +0200 Subject: [PATCH 1/7] add a argument for i18n to enable csv generation --- voxygen/i18n/src/analysis.rs | 9 ++++++++- voxygen/i18n/src/bin/i18n-check.rs | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index dcf8ec3f11..d0328e63a7 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -291,6 +291,7 @@ fn test_localization_directory( ref_manifest: &Path, i18n_references: &HashMap, be_verbose: bool, + csv_enabled: bool, repo: &git2::Repository, head_ref: &git2::Reference, ) -> Option { @@ -494,11 +495,14 @@ fn print_translation_stats( /// `root_dir` - absolute path to main repo /// `assets_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') +/// be_verbose - +/// csv_enabled - generate csv files in target folder pub fn test_specific_localization( code: &str, root_dir: &Path, assets_path: &Path, be_verbose: bool, + csv_enabled: bool, ) { // Relative paths from root of repo to assets let ref_lang_dir = assets_path.join(REFERENCE_LANG); @@ -533,6 +537,7 @@ pub fn test_specific_localization( &ref_manifest, &reference_i18n, be_verbose, + csv_enabled, &repo, &head_ref, ); @@ -542,7 +547,8 @@ pub fn test_specific_localization( /// `root_dir` - absolute path to main repo /// `assets_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') -pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool) { +/// csv_enabled - generate csv files in target folder +pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool, csv_enabled: bool) { let ref_lang_dir = assets_path.join(REFERENCE_LANG); let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); @@ -585,6 +591,7 @@ pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: b &ref_manifest, &reference_i18n, be_verbose, + csv_enabled, &repo, &head_ref, ); diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index cc45d256bb..6af88157e7 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -28,11 +28,17 @@ fn main() { .long("verbose") .help("print additional information"), ) + .arg( + Arg::with_name("csv") + .long("csv") + .help("generate csv files per language in target folder"), + ) .get_matches(); // Generate paths let root = common_assets::find_root().expect("Failed to find root of repository"); let asset_path = Path::new("assets/voxygen/i18n/"); + let csv_enabled = matches.is_present("csv"); if let Some(code) = matches.value_of("CODE") { analysis::test_specific_localization( @@ -40,10 +46,11 @@ fn main() { &root, &asset_path, matches.is_present("verbose"), + csv_enabled, ); } if matches.is_present("test") { - analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose")); + analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose"), csv_enabled); } if matches.is_present("verify") { verification::verify_all_localizations(&root, &asset_path); From c501b2eb705793d6a3d93ba6211058c20fff3ab6 Mon Sep 17 00:00:00 2001 From: "Dr. Dystopia" Date: Fri, 23 Jul 2021 13:32:00 +0200 Subject: [PATCH 2/7] base framework to print csv --- voxygen/i18n/src/analysis.rs | 65 +++-- voxygen/i18n/src/bin/i18n-check.rs | 7 +- voxygen/i18n/src/data.rs | 437 ----------------------------- voxygen/i18n/src/lib.rs | 364 +++++++++++++++++++++++- voxygen/i18n/src/raw.rs | 136 +++++++++ voxygen/i18n/src/verification.rs | 56 ++-- 6 files changed, 570 insertions(+), 495 deletions(-) delete mode 100644 voxygen/i18n/src/data.rs create mode 100644 voxygen/i18n/src/raw.rs diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index d0328e63a7..89841df7d1 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -1,7 +1,7 @@ use ron::de::from_bytes; use std::path::{Path, PathBuf}; -use crate::data::{ +use crate::raw::{ i18n_directories, LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG, }; use hashbrown::{HashMap, HashSet}; @@ -27,6 +27,7 @@ struct LocalizationStats { #[derive(Default)] struct LocalizationAnalysis { + uptodate: Vec<(String, Option)>, notfound: Vec<(String, Option)>, unused: Vec<(String, Option)>, outdated: Vec<(String, Option)>, @@ -39,11 +40,11 @@ impl LocalizationAnalysis { state: LocalizationState, ) -> Option<&mut Vec<(String, Option)>> { match state { + LocalizationState::UpToDate => Some(&mut self.uptodate), LocalizationState::NotFound => Some(&mut self.notfound), LocalizationState::Unused => Some(&mut self.unused), LocalizationState::Outdated => Some(&mut self.outdated), LocalizationState::Unknown => Some(&mut self.unknown), - _ => None, } } @@ -53,9 +54,7 @@ impl LocalizationAnalysis { be_verbose: bool, ref_i18n_map: &HashMap, ) { - let entries = self - .get_mut(state) - .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); + let entries = self.unwrap_entries(state); if entries.is_empty() { return; } @@ -63,9 +62,7 @@ impl LocalizationAnalysis { entries.sort(); for (key, commit_id) in entries { if be_verbose { - let our_commit = commit_id - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()); + let our_commit = LocalizationAnalysis::create_our_commit(commit_id); let ref_commit = ref_i18n_map .get(key) .and_then(|s| s.commit_id) @@ -77,6 +74,32 @@ impl LocalizationAnalysis { } } } + + //TODO: Add which file each faulty translation is in + fn csv(&mut self, state: LocalizationState) { + let entries = self.unwrap_entries(state); + for (key, commit_id) in entries { + let our_commit = LocalizationAnalysis::create_our_commit(commit_id); + println!( + "{},{},{},{:?},{}", + "sv", "_manifest.yml", key, state, our_commit + ); + } + } + + fn unwrap_entries( + &mut self, + state: LocalizationState, + ) -> &mut Vec<(String, Option)> { + self.get_mut(state) + .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)) + } + + fn create_our_commit(commit_id: &mut Option) -> String { + commit_id + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()) + } } #[derive(Copy, Clone, Debug)] @@ -329,14 +352,7 @@ fn test_localization_directory( let mut state_map = LocalizationAnalysis::default(); let result = gather_results(current_i18n, &mut state_map); - print_translation_stats( - i18n_references, - &result, - &mut state_map, - be_verbose, - relfile, - ref_manifest, - ); + print_csv_file(&mut state_map, relfile); Some(result) } @@ -490,6 +506,16 @@ fn print_translation_stats( ); } +fn print_csv_file(state_map: &mut LocalizationAnalysis, relfile: PathBuf) { + println!("country_code,file_name,translation_code,status,git_commit"); + + state_map.csv(LocalizationState::UpToDate); + state_map.csv(LocalizationState::NotFound); + state_map.csv(LocalizationState::Unused); + state_map.csv(LocalizationState::Outdated); + state_map.csv(LocalizationState::Unknown); +} + /// Test one language /// `code` - name of the directory in assets (de_DE for example) /// `root_dir` - absolute path to main repo @@ -548,7 +574,12 @@ pub fn test_specific_localization( /// `assets_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') /// csv_enabled - generate csv files in target folder -pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool, csv_enabled: bool) { +pub fn test_all_localizations( + root_dir: &Path, + assets_path: &Path, + be_verbose: bool, + csv_enabled: bool, +) { let ref_lang_dir = assets_path.join(REFERENCE_LANG); let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index 6af88157e7..f3bdf0a490 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -50,7 +50,12 @@ fn main() { ); } if matches.is_present("test") { - analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose"), csv_enabled); + analysis::test_all_localizations( + &root, + &asset_path, + matches.is_present("verbose"), + csv_enabled, + ); } if matches.is_present("verify") { verification::verify_all_localizations(&root, &asset_path); diff --git a/voxygen/i18n/src/data.rs b/voxygen/i18n/src/data.rs deleted file mode 100644 index 2240134fe2..0000000000 --- a/voxygen/i18n/src/data.rs +++ /dev/null @@ -1,437 +0,0 @@ -use crate::assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; -use deunicode::deunicode; -use hashbrown::{HashMap, HashSet}; -use serde::{Deserialize, Serialize}; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; -use tracing::warn; - -/// The reference language, aka the more up-to-date localization data. -/// Also the default language at first startup. -pub const REFERENCE_LANG: &str = "en"; - -pub const LANG_MANIFEST_FILE: &str = "_manifest"; - -/// How a language can be described -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LanguageMetadata { - /// A human friendly language name (e.g. "English (US)") - pub language_name: String, - - /// A short text identifier for this language (e.g. "en_US") - /// - /// On the opposite of `language_name` that can change freely, - /// `language_identifier` value shall be stable in time as it - /// is used by setting components to store the language - /// selected by the user. - pub language_identifier: String, -} - -/// Store font metadata -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Font { - /// Key to retrieve the font in the asset system - pub asset_key: String, - - /// Scale ratio to resize the UI text dynamicly - scale_ratio: f32, -} - -impl Font { - /// Scale input size to final UI size - pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } -} - -/// Store font metadata -pub type Fonts = HashMap; - -/// Raw localization data, expect the strings to not be loaded here -/// However, metadata informations are correct -/// See `Language` for more info on each attributes -#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] -pub(crate) struct RawLocalization { - pub(crate) convert_utf8_to_ascii: bool, - pub(crate) fonts: Fonts, - pub(crate) metadata: LanguageMetadata, - pub(crate) string_map: HashMap, - pub(crate) vector_map: HashMap>, -} - -/// Store internationalization data -#[derive(Debug, PartialEq, Serialize, Deserialize)] -struct Language { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, - - /// Whether to convert the input text encoded in UTF-8 - /// into a ASCII version by using the `deunicode` crate. - pub(crate) convert_utf8_to_ascii: bool, - - /// Font configuration is stored here - pub(crate) fonts: Fonts, - - pub(crate) metadata: LanguageMetadata, -} - -/// Store internationalization maps -/// These structs are meant to be merged into a Language -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub(crate) struct LocalizationFragment { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, -} - -impl Language { - /// Get a localized text from the given key - pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { - self.string_map.get(key).map(String::as_str) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { - self.vector_map.get(key).and_then(|v| { - if v.is_empty() { - None - } else { - Some(v[index as usize % v.len()].as_str()) - } - }) - } -} - -impl Default for Language { - fn default() -> Self { - Self { - string_map: HashMap::default(), - vector_map: HashMap::default(), - ..Default::default() - } - } -} - -impl From for Language { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - convert_utf8_to_ascii: raw.convert_utf8_to_ascii, - fonts: raw.fonts, - metadata: raw.metadata, - } - } -} -impl From for LocalizationFragment { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - } - } -} - -impl assets::Asset for RawLocalization { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} -impl assets::Asset for LocalizationFragment { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} - -impl assets::Compound for Language { - fn load( - cache: &assets::AssetCache, - asset_key: &str, - ) -> Result { - let raw = cache - .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? - .cloned(); - let mut localization = Language::from(raw); - - // Walk through files in the folder, collecting localization fragment to merge - // inside the asked_localization - for localization_asset in cache - .load_dir::(asset_key, true)? - .iter() - { - localization - .string_map - .extend(localization_asset.read().string_map.clone()); - localization - .vector_map - .extend(localization_asset.read().vector_map.clone()); - } - - // Update the text if UTF-8 to ASCII conversion is enabled - if localization.convert_utf8_to_ascii { - for value in localization.string_map.values_mut() { - *value = deunicode(value); - } - - for value in localization.vector_map.values_mut() { - *value = value.iter().map(|s| deunicode(s)).collect(); - } - } - localization.metadata.language_name = deunicode(&localization.metadata.language_name); - - Ok(localization) - } -} - -/// the central data structure to handle localization in veloren -// inherit Copy+Clone from AssetHandle -#[derive(Debug, PartialEq, Copy, Clone)] -pub struct LocalizationHandle { - active: AssetHandle, - fallback: Option>, - pub use_english_fallback: bool, -} - -// RAII guard returned from Localization::read(), resembles AssetGuard -pub struct LocalizationGuard { - active: AssetGuard, - fallback: Option>, -} - -// arbitrary choice to minimize changing all of veloren -pub type Localization = LocalizationGuard; - -impl LocalizationGuard { - /// Get a localized text from the given key - /// - /// First lookup is done in the active language, second in - /// the fallback (if present). - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get<'a>(&'a self, key: &'a str) -> &str { - self.active.get(key).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get(key)) - .unwrap_or(key) - }) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { - self.active.get_variation(key, index).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get_variation(key, index)) - .unwrap_or(key) - }) - } - - /// Return the missing keys compared to the reference language - fn list_missing_entries(&self) -> (HashSet, HashSet) { - if let Some(ref_lang) = &self.fallback { - let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); - let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); - let strings = reference_string_keys - .difference(&string_keys) - .cloned() - .collect(); - - let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); - let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); - let vectors = reference_vector_keys - .difference(&vector_keys) - .cloned() - .collect(); - - (strings, vectors) - } else { - (HashSet::default(), HashSet::default()) - } - } - - /// Log missing entries (compared to the reference language) as warnings - pub fn log_missing_entries(&self) { - let (missing_strings, missing_vectors) = self.list_missing_entries(); - for missing_key in missing_strings { - warn!( - "[{:?}] Missing string key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - for missing_key in missing_vectors { - warn!( - "[{:?}] Missing vector key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - } - - pub fn fonts(&self) -> &Fonts { &self.active.fonts } - - pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } -} - -impl LocalizationHandle { - pub fn set_english_fallback(&mut self, use_english_fallback: bool) { - self.use_english_fallback = use_english_fallback; - } - - pub fn read(&self) -> LocalizationGuard { - LocalizationGuard { - active: self.active.read(), - fallback: if self.use_english_fallback { - self.fallback.map(|f| f.read()) - } else { - None - }, - } - } - - pub fn load(specifier: &str) -> Result { - let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); - let language_key = ["voxygen.i18n.", specifier].concat(); - let is_default = language_key == default_key; - Ok(Self { - active: Language::load(&language_key)?, - fallback: if is_default { - None - } else { - Language::load(&default_key).ok() - }, - use_english_fallback: false, - }) - } - - pub fn load_expect(specifier: &str) -> Self { - Self::load(specifier).expect("Can't load language files") - } - - pub fn reloaded(&mut self) -> bool { self.active.reloaded() } -} - -struct FindManifests; - -impl assets::Compound for FindManifests { - fn load(_: &assets::AssetCache, _: &str) -> Result { - Ok(Self) - } -} - -impl assets::DirLoadable for FindManifests { - fn select_ids( - source: &S, - specifier: &str, - ) -> io::Result> { - let mut specifiers = Vec::new(); - - source.read_dir(specifier, &mut |entry| { - if let DirEntry::Directory(spec) = entry { - let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); - if source.exists(DirEntry::File(&manifest_spec, "ron")) { - specifiers.push(manifest_spec.into()); - } - } - })?; - - Ok(specifiers) - } -} - -#[derive(Clone, Debug)] -struct LocalizationList(Vec); - -impl assets::Compound for LocalizationList { - fn load( - cache: &assets::AssetCache, - specifier: &str, - ) -> Result { - // List language directories - let languages = assets::load_dir::(specifier, false) - .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) - .ids() - .filter_map(|spec| cache.load::(spec).ok()) - .map(|localization| localization.read().metadata.clone()) - .collect(); - - Ok(LocalizationList(languages)) - } -} - -/// Load all the available languages located in the voxygen asset directory -pub fn list_localizations() -> Vec { - LocalizationList::load_expect_cloned("voxygen.i18n").0 -} - -/// List localization directories as a `PathBuf` vector -pub fn i18n_directories(i18n_dir: &Path) -> Vec { - fs::read_dir(i18n_dir) - .unwrap() - .map(|res| res.map(|e| e.path()).unwrap()) - .filter(|e| e.is_dir()) - .collect() -} - -#[cfg(test)] -mod tests { - use super::assets; - // Test that localization list is loaded (not empty) - #[test] - fn test_localization_list() { - let list = super::list_localizations(); - assert!(!list.is_empty()); - } - - // Test that reference language can be loaded - #[test] - fn test_localization_handle() { - let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); - } - - // Test to verify all languages that they are VALID and loadable, without - // need of git just on the local assets folder - #[test] - fn verify_all_localizations() { - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::verification::verify_all_localizations(&root_dir, i18n_asset_path); - } - - // Test to verify all languages and print missing and faulty localisation - #[test] - #[ignore] - fn test_all_localizations() { - // Options - let be_verbose = true; - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&root_dir, i18n_asset_path, be_verbose); - } -} diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 1d8462e15d..d0b9f85e8c 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,7 +1,365 @@ #[cfg(any(feature = "bin", test))] pub mod analysis; -mod data; +pub mod raw; pub mod verification; -use common_assets as assets; -pub use data::*; +use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; +use hashbrown::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; +use tracing::warn; +use raw::{RawManifest, RawFragment, RawLanguage}; + +/// The reference language, aka the more up-to-date localization data. +/// Also the default language at first startup. +pub const REFERENCE_LANG: &str = "en"; + +pub const LANG_MANIFEST_FILE: &str = "_manifest"; + +pub(crate) const LANG_EXTENSION: &str = "ron"; + +/// How a language can be described +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LanguageMetadata { + /// A human friendly language name (e.g. "English (US)") + pub language_name: String, + + /// A short text identifier for this language (e.g. "en_US") + /// + /// On the opposite of `language_name` that can change freely, + /// `language_identifier` value shall be stable in time as it + /// is used by setting components to store the language + /// selected by the user. + pub language_identifier: String, +} + +/// Store font metadata +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Font { + /// Key to retrieve the font in the asset system + pub asset_key: String, + + /// Scale ratio to resize the UI text dynamically + scale_ratio: f32, +} + +impl Font { + /// Scale input size to final UI size + pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } +} + +/// Store font metadata +pub type Fonts = HashMap; + +/// Store internationalization data +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Language { + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + pub(crate) string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + pub(crate) vector_map: HashMap>, + + /// Whether to convert the input text encoded in UTF-8 + /// into a ASCII version by using the `deunicode` crate. + pub(crate) convert_utf8_to_ascii: bool, + + /// Font configuration is stored here + pub(crate) fonts: Fonts, + + pub(crate) metadata: LanguageMetadata, +} + +impl Language { + /// Get a localized text from the given key + pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { + self.string_map.get(key).map(String::as_str) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { + self.vector_map.get(key).and_then(|v| { + if v.is_empty() { + None + } else { + Some(v[index as usize % v.len()].as_str()) + } + }) + } +} + +impl common_assets::Compound for Language { + fn load( + cache: &common_assets::AssetCache, + asset_key: &str, + ) -> Result { + let manifest = cache + .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? + .cloned(); + + // Walk through files in the folder, collecting localization fragment to merge + // inside the asked_localization + let mut fragments = HashMap::new(); + for fragment_asset in cache + .load_dir::(asset_key, true)? + .iter() + { + let read = fragment_asset.read(); + fragments.insert(PathBuf::from(fragment_asset.id()), read.clone()); + } + + Ok(Language::from(RawLanguage{ + manifest, + fragments, + })) + } +} + +/// the central data structure to handle localization in veloren +// inherit Copy+Clone from AssetHandle +#[derive(Debug, PartialEq, Copy, Clone)] +pub struct LocalizationHandle { + active: AssetHandle, + fallback: Option>, + pub use_english_fallback: bool, +} + +// RAII guard returned from Localization::read(), resembles AssetGuard +pub struct LocalizationGuard { + active: AssetGuard, + fallback: Option>, +} + +// arbitrary choice to minimize changing all of veloren +pub type Localization = LocalizationGuard; + +impl LocalizationGuard { + /// Get a localized text from the given key + /// + /// First lookup is done in the active language, second in + /// the fallback (if present). + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get<'a>(&'a self, key: &'a str) -> &str { + self.active.get(key).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get(key)) + .unwrap_or(key) + }) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { + self.active.get_variation(key, index).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get_variation(key, index)) + .unwrap_or(key) + }) + } + + /// Return the missing keys compared to the reference language + fn list_missing_entries(&self) -> (HashSet, HashSet) { + if let Some(ref_lang) = &self.fallback { + let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); + let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); + let strings = reference_string_keys + .difference(&string_keys) + .cloned() + .collect(); + + let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); + let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); + let vectors = reference_vector_keys + .difference(&vector_keys) + .cloned() + .collect(); + + (strings, vectors) + } else { + (HashSet::default(), HashSet::default()) + } + } + + /// Log missing entries (compared to the reference language) as warnings + pub fn log_missing_entries(&self) { + let (missing_strings, missing_vectors) = self.list_missing_entries(); + for missing_key in missing_strings { + warn!( + "[{:?}] Missing string key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + for missing_key in missing_vectors { + warn!( + "[{:?}] Missing vector key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + } + + pub fn fonts(&self) -> &Fonts { &self.active.fonts } + + pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } +} + +impl LocalizationHandle { + pub fn set_english_fallback(&mut self, use_english_fallback: bool) { + self.use_english_fallback = use_english_fallback; + } + + pub fn read(&self) -> LocalizationGuard { + LocalizationGuard { + active: self.active.read(), + fallback: if self.use_english_fallback { + self.fallback.map(|f| f.read()) + } else { + None + }, + } + } + + pub fn load(specifier: &str) -> Result { + let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); + let language_key = ["voxygen.i18n.", specifier].concat(); + let is_default = language_key == default_key; + Ok(Self { + active: Language::load(&language_key)?, + fallback: if is_default { + None + } else { + Language::load(&default_key).ok() + }, + use_english_fallback: false, + }) + } + + pub fn load_expect(specifier: &str) -> Self { + Self::load(specifier).expect("Can't load language files") + } + + pub fn reloaded(&mut self) -> bool { self.active.reloaded() } +} + +struct FindManifests; + +impl common_assets::Compound for FindManifests { + fn load(_: &common_assets::AssetCache, _: &str) -> Result { + Ok(Self) + } +} + +impl common_assets::DirLoadable for FindManifests { + fn select_ids( + source: &S, + specifier: &str, + ) -> io::Result> { + let mut specifiers = Vec::new(); + + source.read_dir(specifier, &mut |entry| { + if let DirEntry::Directory(spec) = entry { + let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); + if source.exists(DirEntry::File(&manifest_spec, LANG_EXTENSION)) { + specifiers.push(manifest_spec.into()); + } + } + })?; + + Ok(specifiers) + } +} + +#[derive(Clone, Debug)] +struct LocalizationList(Vec); + +impl common_assets::Compound for LocalizationList { + fn load( + cache: &common_assets::AssetCache, + specifier: &str, + ) -> Result { + // List language directories + let languages = common_assets::load_dir::(specifier, false) + .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) + .ids() + .filter_map(|spec| cache.load::(spec).ok()) + .map(|localization| localization.read().metadata.clone()) + .collect(); + + Ok(LocalizationList(languages)) + } +} + +/// Load all the available languages located in the voxygen asset directory +pub fn list_localizations() -> Vec { + LocalizationList::load_expect_cloned("voxygen.i18n").0 +} + +/// List localization directories as a `PathBuf` vector +pub fn i18n_directories(i18n_dir: &Path) -> Vec { + fs::read_dir(i18n_dir) + .unwrap() + .map(|res| res.map(|e| e.path()).unwrap()) + .filter(|e| e.is_dir()) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use common_assets; + + // Test that localization list is loaded (not empty) + #[test] + fn test_localization_list() { + let list = super::list_localizations(); + assert!(!list.is_empty()); + } + + // Test that reference language can be loaded + #[test] + fn test_localization_handle() { + let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); + } + + // Test to verify all languages that they are VALID and loadable, without + // need of git just on the local assets folder + #[test] + fn verify_all_localizations() { + // Generate paths + let i18n_root_path = Path::new("assets/voxygen/i18n/"); + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::verification::verify_all_localizations(&root_dir, i18n_root_path); + } + + // Test to verify all languages and print missing and faulty localisation + #[test] + #[ignore] + fn test_all_localizations() { + // Options + let be_verbose = true; + // Generate paths + let i18n_root_path = Path::new("assets/voxygen/i18n/"); + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::analysis::test_all_localizations(&root_dir, i18n_root_path, be_verbose); + } +} diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs new file mode 100644 index 0000000000..e04d64e706 --- /dev/null +++ b/voxygen/i18n/src/raw.rs @@ -0,0 +1,136 @@ +//! handle the loading of a `Language` +use hashbrown::hash_map::HashMap; +use std::path::{Path, PathBuf}; +use serde::{Deserialize, Serialize}; +use std::fs; +use ron::de::from_reader; +use deunicode::deunicode; +use crate::{Fonts, LanguageMetadata, LANG_MANIFEST_FILE, LANG_EXTENSION}; +use crate::Language; + +/// Raw localization metadata from LANG_MANIFEST_FILE file +/// See `Language` for more info on each attributes +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawManifest { + pub(crate) convert_utf8_to_ascii: bool, + pub(crate) fonts: Fonts, + pub(crate) metadata: LanguageMetadata, +} + +/// Raw localization data from one specific file +/// These structs are meant to be merged into a Language +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawFragment { + pub(crate) string_map: HashMap, + pub(crate) vector_map: HashMap>, +} + +pub(crate) struct RawLanguage { + pub(crate) manifest: RawManifest, + pub(crate) fragments: HashMap, +} + +#[derive(Debug)] +pub(crate) enum RawError { + RonError(ron::Error), +} + +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` folders +pub(crate) fn load_manifest(i18n_root_path: &Path, language_identifier: &str) -> Result { + let manifest_file = i18n_root_path.join(language_identifier).join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); + println!("file , {:?}", manifest_file); + let f = fs::File::open(&manifest_file)?; + Ok(from_reader(f).map_err(RawError::RonError)?) +} + +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` files +pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result { + // Walk through each file in the directory + let mut fragments = HashMap::new(); + let language_identifier = &manifest.metadata.language_identifier; + let language_dir = i18n_root_path.join(language_identifier); + for fragment_file in language_dir.read_dir().unwrap().flatten() { + let file_type = fragment_file.file_type()?; + if file_type.is_dir() { + // TODO: recursive + continue; + } + if file_type.is_file() { + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(&i18n_root_path).unwrap(); + let f = fs::File::open(&full_path)?; + let fragment = from_reader(f).map_err(RawError::RonError)?; + fragments.insert(relative_path.to_path_buf(), fragment); + } + } + Ok(RawLanguage{ + manifest, + fragments, + }) +} + +impl From for Language { + fn from(raw: RawLanguage) -> Self { + + let mut string_map = HashMap::new(); + let mut vector_map = HashMap::new(); + + for (_, fragment) in raw.fragments { + string_map.extend(fragment.string_map); + vector_map.extend(fragment.vector_map); + } + + let convert_utf8_to_ascii = raw.manifest.convert_utf8_to_ascii; + + // Update the text if UTF-8 to ASCII conversion is enabled + if convert_utf8_to_ascii { + for value in string_map.values_mut() { + *value = deunicode(value); + } + + for value in vector_map.values_mut() { + *value = value.iter().map(|s| deunicode(s)).collect(); + } + } + let mut metadata = raw.manifest.metadata; + metadata.language_name = deunicode(&metadata.language_name); + + Self { + string_map, + vector_map, + convert_utf8_to_ascii, + fonts: raw.manifest.fonts, + metadata: metadata, + } + } +} + +impl core::fmt::Display for RawError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + RawError::RonError(e) => write!(f, "{}", e), + } + } +} + +impl std::error::Error for RawError {} + + +impl From for common_assets::Error { + fn from(e: RawError) -> Self { + Self::Conversion(Box::new(e)) + } +} + + +impl common_assets::Asset for RawManifest { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} + +impl common_assets::Asset for RawFragment { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} \ No newline at end of file diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index ad7897c386..9a62974abc 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -1,34 +1,7 @@ -use ron::de::from_reader; -use std::{fs, path::Path}; +use std::{path::Path}; -use crate::data::{i18n_directories, LocalizationFragment, LANG_MANIFEST_FILE, REFERENCE_LANG}; - -fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { - // Walk through each file in the directory - for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - // Skip folders and the manifest file (which does not contain the same struct we - // want to load) - if file_type.is_file() { - let full_path = i18n_file.path(); - println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap()); - let f = fs::File::open(&full_path).expect("Failed opening file"); - let _loc: LocalizationFragment = match from_reader(f) { - Ok(v) => v, - Err(e) => { - panic!( - "Could not parse {} RON file, error: {}", - full_path.to_string_lossy(), - e - ); - }, - }; - } else if file_type.is_dir() { - verify_localization_directory(root_dir, &i18n_file.path()); - } - } - } -} +use crate::{i18n_directories, LANG_MANIFEST_FILE, REFERENCE_LANG}; +use crate::raw; /// Test to verify all languages that they are VALID and loadable, without /// need of git just on the local assets folder @@ -36,17 +9,18 @@ fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { /// `asset_path` - relative path to asset directory (right now it is /// 'assets/voxygen/i18n') pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { - let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG); - let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); + let i18n_root_path = root_dir.join(asset_path); + let ref_i18n_path = i18n_root_path.join(REFERENCE_LANG); + let ref_i18n_manifest_path = ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION); assert!( - root_dir.join(&ref_i18n_dir_path).is_dir(), + root_dir.join(&ref_i18n_path).is_dir(), "Reference language folder doesn't exist, something is wrong!" ); assert!( - root_dir.join(&ref_i18n_path).is_file(), + root_dir.join(&ref_i18n_manifest_path).is_file(), "Reference language manifest file doesn't exist, something is wrong!" ); - let i18n_directories = i18n_directories(&root_dir.join(asset_path)); + let i18n_directories = i18n_directories(&i18n_root_path); // This simple check ONLY guarantees that an arbitrary minimum of translation // files exists. It's just to notice unintentional deletion of all // files, or modifying the paths. In case you want to delete all @@ -57,11 +31,19 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { folder is empty?" ); for i18n_directory in i18n_directories { + let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().as_os_str().to_str().unwrap(); + let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().as_os_str().to_str().unwrap(); println!( "verifying {:?}", - i18n_directory.strip_prefix(&root_dir).unwrap() + display_language_identifier ); // Walk through each files and try to load them - verify_localization_directory(root_dir, &i18n_directory); + verify_localization_directory(root_dir, &asset_path, language_identifier); } } + +fn verify_localization_directory(root_dir: &Path, asset_path: &Path, language_identifier: &str) { + let i18n_path = root_dir.join(asset_path); + let manifest = raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file"); + raw::load_raw_language(&i18n_path, manifest).expect("error accessing fragment file"); +} \ No newline at end of file From a5696e83a961c36de8850de64fce149167d1485e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Tue, 27 Jul 2021 20:07:08 +0200 Subject: [PATCH 3/7] experiment with a git cache --- voxygen/i18n/src/bin/i18n-check.rs | 6 +- voxygen/i18n/src/gitfragments.rs | 172 +++++++++++++++++++++++++++++ voxygen/i18n/src/lib.rs | 6 +- voxygen/i18n/src/raw.rs | 45 ++++---- voxygen/i18n/src/verification.rs | 4 +- 5 files changed, 208 insertions(+), 25 deletions(-) create mode 100644 voxygen/i18n/src/gitfragments.rs diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index f3bdf0a490..174bb0b9d6 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -1,3 +1,4 @@ +/* use clap::{App, Arg}; use std::path::Path; use veloren_i18n::{analysis, verification}; @@ -60,4 +61,7 @@ fn main() { if matches.is_present("verify") { verification::verify_all_localizations(&root, &asset_path); } -} +}*/ + + +fn main() {} \ No newline at end of file diff --git a/voxygen/i18n/src/gitfragments.rs b/voxygen/i18n/src/gitfragments.rs new file mode 100644 index 0000000000..ac415e4ad8 --- /dev/null +++ b/voxygen/i18n/src/gitfragments.rs @@ -0,0 +1,172 @@ +//! fragment attached with git versioning information +use hashbrown::{HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::RwLock; +use std::sync::Arc; +use crate::raw::{RawFragment}; + +struct GitCache<'a> { + pub root_dir: PathBuf, + pub blobs: RwLock>>>, + pub repo: git2::Repository, + //pub head_ref: git2::Reference<'a>, +} + +impl<'a> GitCache<'a> { + pub fn new(root_dir: &Path) -> Self { + let repo = git2::Repository::discover(&root_dir) + .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); + //let head_ref = repo.head().expect("Impossible to get the HEAD reference"); + + let root_dir = root_dir.to_path_buf(); + let blobs = RwLock::new(HashMap::new()); + Self { + root_dir, + blobs, + repo, + //head_ref, + } + } + /// Returns the Git blob associated with the given reference and path + fn read_file_from_path( + &'a self, + reference: &git2::Reference, + path: &std::path::Path, + ) -> Arc> { + // return from cache + let lock = self.blobs.read().unwrap(); + if let Some(blob) = lock.get(path) { + return blob.clone(); + } + drop(lock); + // load file not in cache + let tree = reference + .peel_to_tree() + .expect("Impossible to peel HEAD to a tree object"); + let blob = Arc::new(tree.get_path(path) + .unwrap_or_else(|_| { + panic!( + "Impossible to find the file {:?} in reference {:?}", + path, + reference.name() + ) + }) + .to_object(&self.repo) + .unwrap() + .peel_to_blob() + .expect("Impossible to fetch the Git object")); + let mut lock = self.blobs.write().unwrap(); + let pathbuf = path.to_path_buf(); + lock.insert(pathbuf, blob.clone()); + blob + } +} + +/* +/// Extend a Fragment with historical git data +/// The actual translation gets dropped +fn generate_key_version<'a>( + repo: &'a GitCache, + path: &Path, + fragment: RawFragment, +) -> RawFragment { + let file_blob = repo.read_file_from_path(path); + // Find key start lines + let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); + let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); + for (line_nb, line) in file_content.lines().enumerate() { + let mut found_key = None; + + for key in to_process.iter() { + if correspond(line, key) { + found_key = Some(key.to_owned()); + } + } + + if let Some(key) = found_key { + keys.get_mut(key).unwrap().key_line = Some(line_nb); + to_process.remove(key); + }; + } + + +}*/ + +/* + +fn generate_key_version<'a>( + repo: &'a git2::Repository, + fragment: &RawFragment, + path: &std::path::Path, + file_blob: &git2::Blob, +) -> HashMap { + let mut keys: HashMap = localization + .string_map + .keys() + .map(|k| (k.to_owned(), LocalizationEntryState::new())) + .collect(); + // Find key start lines + let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); + let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); + for (line_nb, line) in file_content.lines().enumerate() { + let mut found_key = None; + + for key in to_process.iter() { + if correspond(line, key) { + found_key = Some(key.to_owned()); + } + } + + if let Some(key) = found_key { + keys.get_mut(key).unwrap().key_line = Some(line_nb); + to_process.remove(key); + }; + } + + let mut error_check_set: Vec = vec![]; + // Find commit for each keys + repo.blame_file(path, None) + .expect("Impossible to generate the Git blame") + .iter() + .for_each(|e: git2::BlameHunk| { + for (key, state) in keys.iter_mut() { + let line = match state.key_line { + Some(l) => l, + None => { + if !error_check_set.contains(key) { + eprintln!( + "Key {} does not have a git line in it's state! Skipping key.", + key + ); + error_check_set.push(key.clone()); + } + continue; + }, + }; + + if line + 1 >= e.final_start_line() + && line + 1 < e.final_start_line() + e.lines_in_hunk() + { + state.chuck_line_range = Some(( + e.final_start_line(), + e.final_start_line() + e.lines_in_hunk(), + )); + state.commit_id = match state.commit_id { + Some(existing_commit) => { + match repo.graph_descendant_of(e.final_commit_id(), existing_commit) { + Ok(true) => Some(e.final_commit_id()), + Ok(false) => Some(existing_commit), + Err(err) => panic!("{}", err), + } + }, + None => Some(e.final_commit_id()), + }; + } + } + }); + + keys +} + + + */ \ No newline at end of file diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index d0b9f85e8c..0c946db4c1 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,5 +1,7 @@ #[cfg(any(feature = "bin", test))] -pub mod analysis; +pub mod gitfragments; +//#[cfg(any(feature = "bin", test))] +//pub mod analysis; pub mod raw; pub mod verification; @@ -113,7 +115,7 @@ impl common_assets::Compound for Language { // inside the asked_localization let mut fragments = HashMap::new(); for fragment_asset in cache - .load_dir::(asset_key, true)? + .load_dir::>(asset_key, true)? .iter() { let read = fragment_asset.read(); diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs index e04d64e706..318bd3bbcd 100644 --- a/voxygen/i18n/src/raw.rs +++ b/voxygen/i18n/src/raw.rs @@ -20,14 +20,14 @@ pub(crate) struct RawManifest { /// Raw localization data from one specific file /// These structs are meant to be merged into a Language #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] -pub(crate) struct RawFragment { - pub(crate) string_map: HashMap, - pub(crate) vector_map: HashMap>, +pub(crate) struct RawFragment { + pub(crate) string_map: HashMap, + pub(crate) vector_map: HashMap>, } -pub(crate) struct RawLanguage { +pub(crate) struct RawLanguage { pub(crate) manifest: RawManifest, - pub(crate) fragments: HashMap, + pub(crate) fragments: HashMap>, } #[derive(Debug)] @@ -44,18 +44,26 @@ pub(crate) fn load_manifest(i18n_root_path: &Path, language_identifier: &str) -> } /// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` files -pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result { +pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result, common_assets::Error> { + let language_identifier = &manifest.metadata.language_identifier; + let fragments = recursive_load_raw_language(i18n_root_path, language_identifier, Path::new(""))?; + Ok(RawLanguage{ + manifest, + fragments, + }) +} + +fn recursive_load_raw_language(i18n_root_path: &Path, language_identifier: &str, subfolder: &Path) -> Result>, common_assets::Error> { // Walk through each file in the directory let mut fragments = HashMap::new(); - let language_identifier = &manifest.metadata.language_identifier; - let language_dir = i18n_root_path.join(language_identifier); - for fragment_file in language_dir.read_dir().unwrap().flatten() { + let search_dir = i18n_root_path.join(language_identifier).join(subfolder); + for fragment_file in search_dir.read_dir().unwrap().flatten() { let file_type = fragment_file.file_type()?; if file_type.is_dir() { - // TODO: recursive - continue; - } - if file_type.is_file() { + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(&search_dir).unwrap(); + fragments.extend(recursive_load_raw_language(i18n_root_path, language_identifier, relative_path)?); + } else if file_type.is_file() { let full_path = fragment_file.path(); let relative_path = full_path.strip_prefix(&i18n_root_path).unwrap(); let f = fs::File::open(&full_path)?; @@ -63,14 +71,11 @@ pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> fragments.insert(relative_path.to_path_buf(), fragment); } } - Ok(RawLanguage{ - manifest, - fragments, - }) + Ok(fragments) } -impl From for Language { - fn from(raw: RawLanguage) -> Self { +impl From> for Language { + fn from(raw: RawLanguage) -> Self { let mut string_map = HashMap::new(); let mut vector_map = HashMap::new(); @@ -129,7 +134,7 @@ impl common_assets::Asset for RawManifest { const EXTENSION: &'static str = LANG_EXTENSION; } -impl common_assets::Asset for RawFragment { +impl common_assets::Asset for RawFragment { type Loader = common_assets::RonLoader; const EXTENSION: &'static str = LANG_EXTENSION; diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index 9a62974abc..89d8146757 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -31,8 +31,8 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { folder is empty?" ); for i18n_directory in i18n_directories { - let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().as_os_str().to_str().unwrap(); - let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().as_os_str().to_str().unwrap(); + let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().to_str().unwrap(); + let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().to_str().unwrap(); println!( "verifying {:?}", display_language_identifier From bdda5ccd72f8d509202c21a1a01f58f50933b1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Wed, 28 Jul 2021 15:20:09 +0200 Subject: [PATCH 4/7] redo i18n git analysis completly and introduce a way to export csv data also improve the performance by caching git_graph_descendant --- voxygen/i18n/src/analysis.rs | 883 ++++++++--------------------- voxygen/i18n/src/bin/i18n-check.rs | 29 +- voxygen/i18n/src/gitfragments.rs | 279 +++++---- voxygen/i18n/src/lib.rs | 28 +- voxygen/i18n/src/raw.rs | 149 +++-- voxygen/i18n/src/stats.rs | 205 +++++++ voxygen/i18n/src/verification.rs | 54 +- 7 files changed, 743 insertions(+), 884 deletions(-) create mode 100644 voxygen/i18n/src/stats.rs diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index 89841df7d1..c336816f28 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -1,686 +1,283 @@ -use ron::de::from_bytes; -use std::path::{Path, PathBuf}; - -use crate::raw::{ - i18n_directories, LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG, +use crate::{ + gitfragments::{ + read_file_from_path, transform_fragment, LocalizationEntryState, LocalizationState, + }, + i18n_directories, + raw::{self, RawFragment, RawLanguage}, + stats::{ + print_csv_file, print_overall_stats, print_translation_stats, LocalizationAnalysis, + LocalizationStats, + }, + REFERENCE_LANG, }; -use hashbrown::{HashMap, HashSet}; +use hashbrown::{hash_map::Entry, HashMap, HashSet}; +use ron::de::from_bytes; +use std::path::Path; -#[derive(Copy, Clone, Eq, Hash, Debug, PartialEq)] -enum LocalizationState { - UpToDate, - NotFound, - Outdated, - Unknown, - Unused, -} - -#[derive(Debug, PartialEq)] -struct LocalizationStats { - uptodate_entries: usize, - outdated_entries: usize, - unused_entries: usize, - notfound_entries: usize, - errors: usize, - real_entry_count: usize, -} - -#[derive(Default)] -struct LocalizationAnalysis { - uptodate: Vec<(String, Option)>, - notfound: Vec<(String, Option)>, - unused: Vec<(String, Option)>, - outdated: Vec<(String, Option)>, - unknown: Vec<(String, Option)>, -} - -impl LocalizationAnalysis { - fn get_mut( - &mut self, - state: LocalizationState, - ) -> Option<&mut Vec<(String, Option)>> { - match state { - LocalizationState::UpToDate => Some(&mut self.uptodate), - LocalizationState::NotFound => Some(&mut self.notfound), - LocalizationState::Unused => Some(&mut self.unused), - LocalizationState::Outdated => Some(&mut self.outdated), - LocalizationState::Unknown => Some(&mut self.unknown), - } - } - - fn show( - &mut self, - state: LocalizationState, - be_verbose: bool, - ref_i18n_map: &HashMap, - ) { - let entries = self.unwrap_entries(state); - if entries.is_empty() { - return; - } - println!("\n\t[{:?}]", state); - entries.sort(); - for (key, commit_id) in entries { - if be_verbose { - let our_commit = LocalizationAnalysis::create_our_commit(commit_id); - let ref_commit = ref_i18n_map - .get(key) - .and_then(|s| s.commit_id) - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()); - println!("{:60}| {:40} | {:40}", key, our_commit, ref_commit,); - } else { - println!("{}", key); - } - } - } - - //TODO: Add which file each faulty translation is in - fn csv(&mut self, state: LocalizationState) { - let entries = self.unwrap_entries(state); - for (key, commit_id) in entries { - let our_commit = LocalizationAnalysis::create_our_commit(commit_id); - println!( - "{},{},{},{:?},{}", - "sv", "_manifest.yml", key, state, our_commit - ); - } - } - - fn unwrap_entries( - &mut self, - state: LocalizationState, - ) -> &mut Vec<(String, Option)> { - self.get_mut(state) - .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)) - } - - fn create_our_commit(commit_id: &mut Option) -> String { - commit_id - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()) - } -} - -#[derive(Copy, Clone, Debug)] -struct LocalizationEntryState { - key_line: Option, - chuck_line_range: Option<(usize, usize)>, - commit_id: Option, - state: LocalizationState, -} - -impl LocalizationEntryState { - fn new() -> LocalizationEntryState { - LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::Unknown, - } - } -} - -/// Returns the Git blob associated with the given reference and path -fn read_file_from_path<'a>( - repo: &'a git2::Repository, - reference: &git2::Reference, - path: &std::path::Path, -) -> git2::Blob<'a> { - let tree = reference - .peel_to_tree() - .expect("Impossible to peel HEAD to a tree object"); - tree.get_path(path) - .unwrap_or_else(|_| { - panic!( - "Impossible to find the file {:?} in reference {:?}", - path, - reference.name() - ) - }) - .to_object(repo) - .unwrap() - .peel_to_blob() - .expect("Impossible to fetch the Git object") -} - -fn correspond(line: &str, key: &str) -> bool { - let pat = { - // Get left part of split - let mut begin = line - .split(':') - .next() - .expect("split always produces value") - .trim() - .chars(); - // Remove quotes - begin.next(); - begin.next_back(); - begin.as_str() - }; - - pat == key -} - -fn generate_key_version<'a>( - repo: &'a git2::Repository, - localization: &LocalizationFragment, - path: &std::path::Path, - file_blob: &git2::Blob, -) -> HashMap { - let mut keys: HashMap = localization - .string_map - .keys() - .map(|k| (k.to_owned(), LocalizationEntryState::new())) - .collect(); - // Find key start lines - let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); - let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); - for (line_nb, line) in file_content.lines().enumerate() { - let mut found_key = None; - - for key in to_process.iter() { - if correspond(line, key) { - found_key = Some(key.to_owned()); - } - } - - if let Some(key) = found_key { - keys.get_mut(key).unwrap().key_line = Some(line_nb); - to_process.remove(key); - }; - } - - let mut error_check_set: Vec = vec![]; - // Find commit for each keys - repo.blame_file(path, None) - .expect("Impossible to generate the Git blame") - .iter() - .for_each(|e: git2::BlameHunk| { - for (key, state) in keys.iter_mut() { - let line = match state.key_line { - Some(l) => l, - None => { - if !error_check_set.contains(key) { - eprintln!( - "Key {} does not have a git line in it's state! Skipping key.", - key - ); - error_check_set.push(key.clone()); - } - continue; - }, - }; - - if line + 1 >= e.final_start_line() - && line + 1 < e.final_start_line() + e.lines_in_hunk() - { - state.chuck_line_range = Some(( - e.final_start_line(), - e.final_start_line() + e.lines_in_hunk(), - )); - state.commit_id = match state.commit_id { - Some(existing_commit) => { - match repo.graph_descendant_of(e.final_commit_id(), existing_commit) { - Ok(true) => Some(e.final_commit_id()), - Ok(false) => Some(existing_commit), - Err(err) => panic!("{}", err), - } - }, - None => Some(e.final_commit_id()), - }; - } - } - }); - - keys -} - -fn complete_key_versions<'a>( +/// Fill the entry State base information (except `state`) for a complete +/// language +fn gather_entry_state<'a>( repo: &'a git2::Repository, head_ref: &git2::Reference, - i18n_key_versions: &mut HashMap, - root_dir: &Path, - lang_dir: &Path, -) { - //TODO: review unwraps in this file + language_identifier: &str, + root_path: &Path, + relative_i18n_root_path: &Path, +) -> RawLanguage { + println!("-> {:?}", &language_identifier); + let i18n_root_path = root_path.join(relative_i18n_root_path); + // load standard manifest + let manifest = raw::load_manifest(&i18n_root_path, language_identifier) + .expect("failed to load language manifest"); + // transform language into LocalizationEntryState + let mut fragments = HashMap::new(); // For each file in directory - for i18n_file in root_dir.join(&lang_dir).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - if file_type.is_file() { - println!("-> {:?}", i18n_file.file_name()); + let files = raw::fragments_pathes_in_language(&i18n_root_path, language_identifier) + .expect("failed to get all files in language"); + for subpath in files { + let path = relative_i18n_root_path + .join(language_identifier) + .join(&subpath); + println!(" -> {:?}", &subpath); + let i18n_blob = read_file_from_path(repo, head_ref, &path); + let fragment: RawFragment = from_bytes(i18n_blob.content()).unwrap_or_else(|e| { + panic!( + "Could not parse {} RON file, skipping: {}", + subpath.to_string_lossy(), + e + ) + }); + let frag = transform_fragment(repo, (&path, fragment), &i18n_blob); + fragments.insert(subpath.to_path_buf(), frag); + } - let full_path = i18n_file.path(); - let path = full_path.strip_prefix(root_dir).unwrap(); - let i18n_blob = read_file_from_path(repo, head_ref, path); - let i18n: LocalizationFragment = - from_bytes(i18n_blob.content()).unwrap_or_else(|e| { - panic!( - "Could not parse {} RON file, skipping: {}", - i18n_file.path().to_string_lossy(), - e - ) - }); - i18n_key_versions.extend(generate_key_version(repo, &i18n, path, &i18n_blob)); - } else if file_type.is_dir() { - // If it's a directory, recursively check it - complete_key_versions( - repo, - head_ref, - i18n_key_versions, - root_dir, - &i18n_file.path(), + RawLanguage:: { + manifest, + fragments, + } +} + +/// fills in the `state` +fn compare_lang_with_reference( + current_i18n: &mut RawLanguage, + i18n_references: &RawLanguage, + repo: &git2::Repository, +) { + // git graph decendent of is slow, so we cache it + let mut graph_decendent_of_cache = HashMap::new(); + + let mut cached_graph_descendant_of = |commit, ancestor| -> bool { + let key = (commit, ancestor); + match graph_decendent_of_cache.entry(key) { + Entry::Occupied(entry) => { + return *entry.get(); + }, + Entry::Vacant(entry) => { + let value = repo.graph_descendant_of(commit, ancestor).unwrap_or(false); + *entry.insert(value) + }, + } + }; + + // match files + for (ref_path, ref_fragment) in i18n_references.fragments.iter() { + let cur_fragment = match current_i18n.fragments.get_mut(ref_path) { + Some(c) => c, + None => { + eprintln!( + "language {} is missing file: {:?}", + current_i18n.manifest.metadata.language_identifier, ref_path ); + continue; + }, + }; + + for (ref_key, ref_state) in ref_fragment.string_map.iter() { + match cur_fragment.string_map.get_mut(ref_key) { + Some(state) => { + let commit_id = match state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in i18n file {} is missing! Skipping key.", + ref_key, + ref_path.to_string_lossy() + ); + continue; + }, + }; + let ref_commit_id = match ref_state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in reference i18n file is missing! Skipping \ + key.", + ref_key + ); + continue; + }, + }; + if commit_id != ref_commit_id + && !cached_graph_descendant_of(commit_id, ref_commit_id) + { + state.state = Some(LocalizationState::Outdated); + } else { + state.state = Some(LocalizationState::UpToDate); + } + }, + None => { + cur_fragment + .string_map + .insert(ref_key.to_owned(), LocalizationEntryState { + key_line: None, + chuck_line_range: None, + commit_id: None, + state: Some(LocalizationState::NotFound), + }); + }, } } - } -} -fn gather_state( - loc: &RawLocalization, - i18n_blob: &git2::Blob, - ref_manifest: &Path, - root_dir: &Path, - lang_dir: &Path, - repo: &git2::Repository, - head_ref: &git2::Reference, -) -> HashMap { - // Generate map - let mut i18n_map = generate_key_version( - repo, - &LocalizationFragment::from(loc.clone()), - ref_manifest, - i18n_blob, - ); - - // Gathering info about keys from language - complete_key_versions(repo, head_ref, &mut i18n_map, root_dir, lang_dir); - - i18n_map -} - -// Helper function to test localization directory -// `lang_dir` - path to localization directory. Relative from root of the -// repo. -// `root_dir` - absolute path to repo -// `ref_manifest` - path to reference manifest -// `i18n_references` - keys from reference language -// `repo` - git object for main repo -// `head_ref` - HEAD -fn test_localization_directory( - lang_dir: &Path, - root_dir: &Path, - ref_manifest: &Path, - i18n_references: &HashMap, - be_verbose: bool, - csv_enabled: bool, - repo: &git2::Repository, - head_ref: &git2::Reference, -) -> Option { - let relfile = lang_dir.join(&(LANG_MANIFEST_FILE.to_string() + ".ron")); - if relfile == ref_manifest { - return None; - } - println!("\n-----------------------------------"); - println!("{:?}", relfile); - println!("-----------------------------------"); - - // Find the localization entry state - let current_blob = read_file_from_path(repo, head_ref, &relfile); - let current_loc: RawLocalization = from_bytes(current_blob.content()).unwrap_or_else(|e| { - panic!( - "Could not parse {} RON file, skipping: {}", - relfile.to_string_lossy(), - e - ) - }); - - // Gather state of current localization - let mut current_i18n = gather_state( - ¤t_loc, - ¤t_blob, - ref_manifest, - root_dir, - lang_dir, - repo, - head_ref, - ); - - // Comparing with reference localization - fill_info(&mut current_i18n, i18n_references, repo, &relfile); - - let mut state_map = LocalizationAnalysis::default(); - let result = gather_results(current_i18n, &mut state_map); - print_csv_file(&mut state_map, relfile); - Some(result) -} - -fn fill_info( - current_i18n: &mut HashMap, - i18n_references: &HashMap, - repo: &git2::Repository, - relfile: &Path, -) { - for (ref_key, ref_state) in i18n_references.iter() { - match current_i18n.get_mut(ref_key) { - Some(state) => { - let commit_id = match state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in i18n file {} is missing! Skipping key.", - ref_key, - relfile.to_string_lossy() - ); - continue; - }, - }; - let ref_commit_id = match ref_state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in reference i18n file is missing! Skipping key.", - ref_key - ); - continue; - }, - }; - if commit_id != ref_commit_id - && !repo - .graph_descendant_of(commit_id, ref_commit_id) - .unwrap_or(false) - { - state.state = LocalizationState::Outdated; - } else { - state.state = LocalizationState::UpToDate; - } - }, - None => { - current_i18n.insert(ref_key.to_owned(), LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::NotFound, - }); - }, + let ref_keys: HashSet<&String> = ref_fragment.string_map.keys().collect(); + for (_, state) in cur_fragment + .string_map + .iter_mut() + .filter(|&(k, _)| !ref_keys.contains(k)) + { + state.state = Some(LocalizationState::Unused); } } - - let ref_keys: HashSet<&String> = i18n_references.keys().collect(); - for (_, state) in current_i18n - .iter_mut() - .filter(|&(k, _)| !ref_keys.contains(k)) - { - state.state = LocalizationState::Unused; - } } fn gather_results( - current_i18n: HashMap, - state_map: &mut LocalizationAnalysis, -) -> LocalizationStats { - let mut uptodate_entries = 0; - let mut outdated_entries = 0; - let mut unused_entries = 0; - let mut notfound_entries = 0; - let mut unknown_entries = 0; + current_i18n: &RawLanguage, +) -> (LocalizationAnalysis, LocalizationStats) { + let mut state_map = + LocalizationAnalysis::new(¤t_i18n.manifest.metadata.language_identifier); + let mut stats = LocalizationStats::default(); - let keys: Vec<&String> = current_i18n.keys().collect(); - for key in keys { - let entry = current_i18n.get(key).unwrap(); - match entry.state { - LocalizationState::Outdated => outdated_entries += 1, - LocalizationState::NotFound => notfound_entries += 1, - LocalizationState::Unknown => unknown_entries += 1, - LocalizationState::Unused => unused_entries += 1, - LocalizationState::UpToDate => uptodate_entries += 1, - }; - if entry.state != LocalizationState::UpToDate { - let state_keys = state_map - .get_mut(entry.state) - .expect("vectors must be added"); - state_keys.push((key.to_owned(), entry.commit_id)); + for (file, fragments) in ¤t_i18n.fragments { + for (key, entry) in &fragments.string_map { + match entry.state { + Some(LocalizationState::Outdated) => stats.outdated_entries += 1, + Some(LocalizationState::NotFound) => stats.notfound_entries += 1, + None => stats.errors += 1, + Some(LocalizationState::Unused) => stats.unused_entries += 1, + Some(LocalizationState::UpToDate) => stats.uptodate_entries += 1, + }; + if entry.state != Some(LocalizationState::UpToDate) { + let state_keys = state_map.data.get_mut(&entry.state).expect("prefiled"); + state_keys.push((file.clone(), key.to_owned(), entry.commit_id)); + } } } - // Calculate key count that actually matter for the status of the translation - // Unused entries don't break the game - let current_i18n_entry_count = current_i18n.len(); - let real_entry_count = current_i18n_entry_count - unused_entries; - - LocalizationStats { - uptodate_entries, - unused_entries, - outdated_entries, - notfound_entries, - errors: unknown_entries, - real_entry_count, + for (_, entries) in state_map.data.iter_mut() { + entries.sort(); } + + (state_map, stats) } -fn print_translation_stats( - ref_i18n_map: &HashMap, - stats: &LocalizationStats, - state_map: &mut LocalizationAnalysis, - be_verbose: bool, - relfile: PathBuf, - ref_manifest: &Path, +/// completely analysis multiple languages without printing +fn complete_analysis( + language_identifiers: &[&str], + root_path: &Path, + relative_i18n_root_path: &Path, +) -> ( + HashMap, + /* ref lang */ RawLanguage, ) { - let uptodate_percent = - (stats.uptodate_entries as f32 / stats.real_entry_count as f32) * 100_f32; - let outdated_percent = - (stats.outdated_entries as f32 / stats.real_entry_count as f32) * 100_f32; - let untranslated_percent = - ((stats.errors + stats.errors) as f32 / stats.real_entry_count as f32) * 100_f32; + let mut result = HashMap::new(); + // Initialize Git objects + let repo = git2::Repository::discover(&root_path) + .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_path)); + let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - // Display - if be_verbose { - println!( - "\n{:60}| {:40} | {:40}", - "Key name", - relfile.to_str().unwrap(), - ref_manifest.to_str().unwrap(), + // Read Reference Language + let ref_language = gather_entry_state( + &repo, + &head_ref, + REFERENCE_LANG, + root_path, + relative_i18n_root_path, + ); + for &language_identifier in language_identifiers { + let mut cur_language = gather_entry_state( + &repo, + &head_ref, + language_identifier, + root_path, + relative_i18n_root_path, ); - } else { - println!("\nKey name"); + compare_lang_with_reference(&mut cur_language, &ref_language, &repo); + let (state_map, stats) = gather_results(&cur_language); + result.insert(language_identifier.to_owned(), (state_map, stats)); } - - state_map.show(LocalizationState::NotFound, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Unused, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Outdated, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Unknown, be_verbose, ref_i18n_map); - - println!( - "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", - stats.uptodate_entries, - stats.outdated_entries, - stats.unused_entries, - stats.notfound_entries, - stats.errors, - ); - - println!( - "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", - uptodate_percent, outdated_percent, untranslated_percent, - ); -} - -fn print_csv_file(state_map: &mut LocalizationAnalysis, relfile: PathBuf) { - println!("country_code,file_name,translation_code,status,git_commit"); - - state_map.csv(LocalizationState::UpToDate); - state_map.csv(LocalizationState::NotFound); - state_map.csv(LocalizationState::Unused); - state_map.csv(LocalizationState::Outdated); - state_map.csv(LocalizationState::Unknown); + (result, ref_language) } /// Test one language -/// `code` - name of the directory in assets (de_DE for example) -/// `root_dir` - absolute path to main repo -/// `assets_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -/// be_verbose - -/// csv_enabled - generate csv files in target folder -pub fn test_specific_localization( - code: &str, - root_dir: &Path, - assets_path: &Path, +/// - `code`: name of the directory in assets (de_DE for example) +/// - `root_path`: absolute path to main repo +/// - `relative_i18n_root_path`: relative path to asset directory (right now it +/// is 'assets/voxygen/i18n') +/// - be_verbose: print extra info +/// - csv_enabled: generate csv files in target folder +pub fn test_specific_localizations( + language_identifiers: &[&str], + root_path: &Path, + relative_i18n_root_path: &Path, be_verbose: bool, csv_enabled: bool, ) { - // Relative paths from root of repo to assets - let ref_lang_dir = assets_path.join(REFERENCE_LANG); - let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - - // Initialize Git objects - let repo = git2::Repository::discover(&root_dir) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); - let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - - // Read HEAD for the reference language manifest - let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest); - let loc: RawLocalization = from_bytes(ref_manifest_blob.content()) - .expect("Expect to parse reference i18n RON file, can't proceed without it"); - - // Gathering info about keys from reference language - let reference_i18n = gather_state( - &loc, - &ref_manifest_blob, - &ref_manifest, - root_dir, - &ref_lang_dir, - &repo, - &head_ref, - ); - - // Testing how specific language is localized - let dir = assets_path.join(code); - test_localization_directory( - &dir, - root_dir, - &ref_manifest, - &reference_i18n, - be_verbose, - csv_enabled, - &repo, - &head_ref, - ); + let (analysis, reference_language) = + complete_analysis(language_identifiers, root_path, relative_i18n_root_path); + for (language_identifier, (state_map, stats)) in &analysis { + if csv_enabled { + print_csv_file(state_map); + } else { + print_translation_stats( + language_identifier, + &reference_language, + stats, + state_map, + be_verbose, + ); + } + } + if analysis.len() > 1 { + print_overall_stats(analysis); + } } /// Test all localizations -/// `root_dir` - absolute path to main repo -/// `assets_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -/// csv_enabled - generate csv files in target folder pub fn test_all_localizations( - root_dir: &Path, - assets_path: &Path, + root_path: &Path, + relative_i18n_root_path: &Path, be_verbose: bool, csv_enabled: bool, ) { - let ref_lang_dir = assets_path.join(REFERENCE_LANG); - let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - - if !root_dir.join(&ref_lang_dir).is_dir() { - panic!("Reference language folder not found {:?}", &ref_lang_dir) - } - if !root_dir.join(&ref_manifest).is_file() { - panic!("Reference language file not found {:?}", &ref_manifest) - } - - // Initialize Git objects - let repo = git2::Repository::discover(&root_dir) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); - let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - - // Read HEAD for the reference language file - let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest); - let loc: RawLocalization = from_bytes(ref_manifest_blob.content()) - .expect("Expect to parse reference i18n RON file, can't proceed without it"); - - // Gathering info about keys from reference language - let reference_i18n = gather_state( - &loc, - &ref_manifest_blob, - &ref_manifest, - root_dir, - &ref_lang_dir, - &repo, - &head_ref, - ); - + let i18n_root_path = root_path.join(relative_i18n_root_path); // Compare to other reference files - let i18n_directories = i18n_directories(&root_dir.join(assets_path)); - let mut i18n_entry_counts: HashMap = HashMap::new(); - for dir in &i18n_directories { - let rel_dir = dir.strip_prefix(root_dir).unwrap(); - let result = test_localization_directory( - rel_dir, - root_dir, - &ref_manifest, - &reference_i18n, - be_verbose, - csv_enabled, - &repo, - &head_ref, - ); - if let Some(values) = result { - i18n_entry_counts.insert(dir.clone(), values); - } - } - - print_overall_stats(i18n_entry_counts); -} - -fn print_overall_stats(i18n_entry_counts: HashMap) { - let mut overall_uptodate_entry_count = 0; - let mut overall_outdated_entry_count = 0; - let mut overall_untranslated_entry_count = 0; - let mut overall_real_entry_count = 0; - - println!("-----------------------------------------------------------------------------"); - println!("Overall Translation Status"); - println!("-----------------------------------------------------------------------------"); - println!( - "{:12}| {:8} | {:8} | {:8} | {:8} | {:8}", - "", "up-to-date", "outdated", "untranslated", "unused", "errors", + let language_identifiers = i18n_directories(&i18n_root_path) + .into_iter() + .map(|p| { + p.strip_prefix(&i18n_root_path) + .unwrap() + .to_str() + .unwrap() + .to_owned() + }) + .collect::>(); + test_specific_localizations( + &language_identifiers + .iter() + .map(|s| s.as_str()) + .collect::>(), + root_path, + relative_i18n_root_path, + be_verbose, + csv_enabled, ); - - let mut i18n_stats: Vec<(&PathBuf, &LocalizationStats)> = i18n_entry_counts.iter().collect(); - i18n_stats.sort_by_key(|(_, result)| result.notfound_entries); - - for (path, test_result) in i18n_stats { - let LocalizationStats { - uptodate_entries: uptodate, - outdated_entries: outdated, - unused_entries: unused, - notfound_entries: untranslated, - errors, - real_entry_count: real, - } = test_result; - overall_uptodate_entry_count += uptodate; - overall_outdated_entry_count += outdated; - overall_untranslated_entry_count += untranslated; - overall_real_entry_count += real; - - println!( - "{:12}|{:8} |{:6} |{:8} |{:6} |{:8}", - path.file_name().unwrap().to_string_lossy(), - uptodate, - outdated, - untranslated, - unused, - errors, - ); - } - - println!( - "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", - (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - ); - println!("-----------------------------------------------------------------------------\n"); } diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index 174bb0b9d6..9972850b15 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -1,4 +1,3 @@ -/* use clap::{App, Arg}; use std::path::Path; use veloren_i18n::{analysis, verification}; @@ -37,31 +36,29 @@ fn main() { .get_matches(); // Generate paths - let root = common_assets::find_root().expect("Failed to find root of repository"); - let asset_path = Path::new("assets/voxygen/i18n/"); + let root_path = common_assets::find_root().expect("Failed to find root of repository"); + let relative_i18n_root_path = Path::new("assets/voxygen/i18n/"); + let be_verbose = matches.is_present("verbose"); let csv_enabled = matches.is_present("csv"); if let Some(code) = matches.value_of("CODE") { - analysis::test_specific_localization( - code, - &root, - &asset_path, - matches.is_present("verbose"), + analysis::test_specific_localizations( + &[code], + &root_path, + relative_i18n_root_path, + be_verbose, csv_enabled, ); } if matches.is_present("test") { analysis::test_all_localizations( - &root, - &asset_path, - matches.is_present("verbose"), + &root_path, + relative_i18n_root_path, + be_verbose, csv_enabled, ); } if matches.is_present("verify") { - verification::verify_all_localizations(&root, &asset_path); + verification::verify_all_localizations(&root_path, relative_i18n_root_path); } -}*/ - - -fn main() {} \ No newline at end of file +} diff --git a/voxygen/i18n/src/gitfragments.rs b/voxygen/i18n/src/gitfragments.rs index ac415e4ad8..e517360d42 100644 --- a/voxygen/i18n/src/gitfragments.rs +++ b/voxygen/i18n/src/gitfragments.rs @@ -1,172 +1,157 @@ //! fragment attached with git versioning information -use hashbrown::{HashMap}; -use std::path::{Path, PathBuf}; -use std::sync::RwLock; -use std::sync::Arc; -use crate::raw::{RawFragment}; +use crate::raw::RawFragment; +use hashbrown::HashMap; +use std::path::Path; -struct GitCache<'a> { - pub root_dir: PathBuf, - pub blobs: RwLock>>>, - pub repo: git2::Repository, - //pub head_ref: git2::Reference<'a>, +#[derive(Copy, Clone, Eq, Hash, Debug, PartialEq)] +pub(crate) enum LocalizationState { + UpToDate, + NotFound, + Outdated, + Unused, } -impl<'a> GitCache<'a> { - pub fn new(root_dir: &Path) -> Self { - let repo = git2::Repository::discover(&root_dir) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); - //let head_ref = repo.head().expect("Impossible to get the HEAD reference"); +pub(crate) const ALL_LOCALIZATION_STATES: [Option; 5] = [ + Some(LocalizationState::UpToDate), + Some(LocalizationState::NotFound), + Some(LocalizationState::Outdated), + Some(LocalizationState::Unused), + None, +]; - let root_dir = root_dir.to_path_buf(); - let blobs = RwLock::new(HashMap::new()); - Self { - root_dir, - blobs, - repo, - //head_ref, +#[derive(Copy, Clone, Debug)] +pub(crate) struct LocalizationEntryState { + pub(crate) key_line: Option, + pub(crate) chuck_line_range: Option<(usize, usize)>, + pub(crate) commit_id: Option, + pub(crate) state: Option, +} + +impl LocalizationState { + pub(crate) fn print(this: &Option) -> String { + match this { + Some(LocalizationState::UpToDate) => "UpToDate", + Some(LocalizationState::NotFound) => "NotFound", + Some(LocalizationState::Outdated) => "Outdated", + Some(LocalizationState::Unused) => "Unused", + None => "Unknown", } - } - /// Returns the Git blob associated with the given reference and path - fn read_file_from_path( - &'a self, - reference: &git2::Reference, - path: &std::path::Path, - ) -> Arc> { - // return from cache - let lock = self.blobs.read().unwrap(); - if let Some(blob) = lock.get(path) { - return blob.clone(); - } - drop(lock); - // load file not in cache - let tree = reference - .peel_to_tree() - .expect("Impossible to peel HEAD to a tree object"); - let blob = Arc::new(tree.get_path(path) - .unwrap_or_else(|_| { - panic!( - "Impossible to find the file {:?} in reference {:?}", - path, - reference.name() - ) - }) - .to_object(&self.repo) - .unwrap() - .peel_to_blob() - .expect("Impossible to fetch the Git object")); - let mut lock = self.blobs.write().unwrap(); - let pathbuf = path.to_path_buf(); - lock.insert(pathbuf, blob.clone()); - blob + .to_owned() } } -/* +impl LocalizationEntryState { + fn new(key_line: Option) -> LocalizationEntryState { + LocalizationEntryState { + key_line, + chuck_line_range: None, + commit_id: None, + state: None, + } + } +} + +/// Returns the Git blob associated with the given reference and path +pub(crate) fn read_file_from_path<'a>( + repo: &'a git2::Repository, + reference: &git2::Reference, + path: &std::path::Path, +) -> git2::Blob<'a> { + let tree = reference + .peel_to_tree() + .expect("Impossible to peel HEAD to a tree object"); + tree.get_path(path) + .unwrap_or_else(|_| { + panic!( + "Impossible to find the file {:?} in reference {:?}", + path, + reference.name() + ) + }) + .to_object(repo) + .unwrap() + .peel_to_blob() + .expect("Impossible to fetch the Git object") +} + /// Extend a Fragment with historical git data /// The actual translation gets dropped -fn generate_key_version<'a>( - repo: &'a GitCache, - path: &Path, - fragment: RawFragment, -) -> RawFragment { - let file_blob = repo.read_file_from_path(path); - // Find key start lines - let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); - let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); - for (line_nb, line) in file_content.lines().enumerate() { - let mut found_key = None; - - for key in to_process.iter() { - if correspond(line, key) { - found_key = Some(key.to_owned()); - } - } - - if let Some(key) = found_key { - keys.get_mut(key).unwrap().key_line = Some(line_nb); - to_process.remove(key); - }; - } - - -}*/ - -/* - -fn generate_key_version<'a>( +/// TODO: transform vector_map too +pub(crate) fn transform_fragment<'a>( repo: &'a git2::Repository, - fragment: &RawFragment, - path: &std::path::Path, + fragment: (&Path, RawFragment), file_blob: &git2::Blob, -) -> HashMap { - let mut keys: HashMap = localization - .string_map - .keys() - .map(|k| (k.to_owned(), LocalizationEntryState::new())) - .collect(); - // Find key start lines +) -> RawFragment { + let (path, fragment) = fragment; + // Find key start lines by searching all lines which have `:` in them (as they + // are probably keys) and getting the first part of such line trimming + // whitespace and quotes. Quite buggy heuristic let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); - let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); - for (line_nb, line) in file_content.lines().enumerate() { - let mut found_key = None; + // we only need the key part of the file to process + let file_content_keys = file_content.lines().enumerate().filter_map(|(no, line)| { + line.split_once(':').map(|(key, _)| { + let mut key = key.trim().chars(); + key.next(); + key.next_back(); + (no, key.as_str()) + }) + }); + //speed up the search by sorting all keys! + let mut file_content_keys_sorted = file_content_keys.into_iter().collect::>(); + file_content_keys_sorted.sort_by_key(|(_, key)| *key); - for key in to_process.iter() { - if correspond(line, key) { - found_key = Some(key.to_owned()); - } - } + let mut result = RawFragment:: { + string_map: HashMap::new(), + vector_map: HashMap::new(), + }; - if let Some(key) = found_key { - keys.get_mut(key).unwrap().key_line = Some(line_nb); - to_process.remove(key); - }; + for (original_key, _) in fragment.string_map { + let line_nb = file_content_keys_sorted + .binary_search_by_key(&original_key.as_str(), |(_, key)| *key) + .map_or_else( + |_| { + eprintln!( + "Key {} does not have a git line in it's state!", + original_key + ); + None + }, + |id| Some(file_content_keys_sorted[id].0), + ); + + result + .string_map + .insert(original_key, LocalizationEntryState::new(line_nb)); } - let mut error_check_set: Vec = vec![]; - // Find commit for each keys - repo.blame_file(path, None) + // Find commit for each keys, THIS PART IS SLOW (2s/4s) + for e in repo + .blame_file(path, None) .expect("Impossible to generate the Git blame") .iter() - .for_each(|e: git2::BlameHunk| { - for (key, state) in keys.iter_mut() { - let line = match state.key_line { - Some(l) => l, - None => { - if !error_check_set.contains(key) { - eprintln!( - "Key {} does not have a git line in it's state! Skipping key.", - key - ); - error_check_set.push(key.clone()); - } - continue; - }, - }; - - if line + 1 >= e.final_start_line() - && line + 1 < e.final_start_line() + e.lines_in_hunk() - { - state.chuck_line_range = Some(( - e.final_start_line(), - e.final_start_line() + e.lines_in_hunk(), - )); - state.commit_id = match state.commit_id { - Some(existing_commit) => { - match repo.graph_descendant_of(e.final_commit_id(), existing_commit) { - Ok(true) => Some(e.final_commit_id()), - Ok(false) => Some(existing_commit), - Err(err) => panic!("{}", err), - } + { + for (_, state) in result.string_map.iter_mut() { + if let Some(line) = state.key_line { + let range = ( + e.final_start_line(), + e.final_start_line() + e.lines_in_hunk(), + ); + if line + 1 >= range.0 && line + 1 < range.1 { + state.chuck_line_range = Some(range); + state.commit_id = state.commit_id.map_or_else( + || Some(e.final_commit_id()), + |existing_commit| match repo + .graph_descendant_of(e.final_commit_id(), existing_commit) + { + Ok(true) => Some(e.final_commit_id()), + Ok(false) => Some(existing_commit), + Err(err) => panic!("{}", err), }, - None => Some(e.final_commit_id()), - }; + ); } } - }); + } + } - keys + result } - - - */ \ No newline at end of file diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 0c946db4c1..9360823ee1 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,19 +1,20 @@ #[cfg(any(feature = "bin", test))] +pub mod analysis; +#[cfg(any(feature = "bin", test))] pub mod gitfragments; -//#[cfg(any(feature = "bin", test))] -//pub mod analysis; pub mod raw; +#[cfg(any(feature = "bin", test))] pub mod stats; pub mod verification; use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; use hashbrown::{HashMap, HashSet}; +use raw::{RawFragment, RawLanguage, RawManifest}; use serde::{Deserialize, Serialize}; use std::{ fs, io, path::{Path, PathBuf}, }; use tracing::warn; -use raw::{RawManifest, RawFragment, RawLanguage}; /// The reference language, aka the more up-to-date localization data. /// Also the default language at first startup. @@ -118,11 +119,18 @@ impl common_assets::Compound for Language { .load_dir::>(asset_key, true)? .iter() { + let id = fragment_asset.id(); + // Activate this once ._manifest is fully transformed and only contains metadata + // or regex: "(_: &common_assets::AssetCache, _: &str) -> Result { + fn load( + _: &common_assets::AssetCache, + _: &str, + ) -> Result { Ok(Self) } } @@ -328,7 +339,6 @@ pub fn i18n_directories(i18n_dir: &Path) -> Vec { #[cfg(test)] mod tests { use std::path::Path; - use common_assets; // Test that localization list is loaded (not empty) #[test] @@ -357,11 +367,9 @@ mod tests { #[test] #[ignore] fn test_all_localizations() { - // Options - let be_verbose = true; // Generate paths let i18n_root_path = Path::new("assets/voxygen/i18n/"); let root_dir = common_assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&root_dir, i18n_root_path, be_verbose); + crate::analysis::test_all_localizations(&root_dir, i18n_root_path, true, false); } } diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs index 318bd3bbcd..e3e00a8d4e 100644 --- a/voxygen/i18n/src/raw.rs +++ b/voxygen/i18n/src/raw.rs @@ -1,12 +1,24 @@ //! handle the loading of a `Language` -use hashbrown::hash_map::HashMap; -use std::path::{Path, PathBuf}; -use serde::{Deserialize, Serialize}; -use std::fs; -use ron::de::from_reader; +//! Paths: +//! - `root_path`: repo part, git main folder +//! - `language_identifier`: `en`, `de_DE`, `fr_FR`, etc.. +//! - `relative_i18n_root_path`: relative path to i18n path which contains +//! `language_identifier` folders from `root_path` +//! - `i18n_root_path`: absolute path to `relative_i18n_root_path` +//! - `i18n_path`: absolute path to `i18n_root_path` + `language_identifier` +//! - `subfolder`: all folders in `i18n_path` +//! +//! wherever possible we use relative paths only. So expect 1 absolute +//! `root_path` or `i18n_root_path` to be required and all others be relative. +use crate::{Fonts, Language, LanguageMetadata, LANG_EXTENSION, LANG_MANIFEST_FILE}; use deunicode::deunicode; -use crate::{Fonts, LanguageMetadata, LANG_MANIFEST_FILE, LANG_EXTENSION}; -use crate::Language; +use hashbrown::hash_map::HashMap; +use ron::de::from_reader; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + path::{Path, PathBuf}, +}; /// Raw localization metadata from LANG_MANIFEST_FILE file /// See `Language` for more info on each attributes @@ -27,7 +39,7 @@ pub(crate) struct RawFragment { pub(crate) struct RawLanguage { pub(crate) manifest: RawManifest, - pub(crate) fragments: HashMap>, + pub(crate) fragments: HashMap>, } #[derive(Debug)] @@ -35,48 +47,52 @@ pub(crate) enum RawError { RonError(ron::Error), } -/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` folders -pub(crate) fn load_manifest(i18n_root_path: &Path, language_identifier: &str) -> Result { - let manifest_file = i18n_root_path.join(language_identifier).join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); - println!("file , {:?}", manifest_file); +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, +/// `fr_FR` folders +pub(crate) fn load_manifest( + i18n_root_path: &Path, + language_identifier: &str, +) -> Result { + let manifest_file = i18n_root_path + .join(language_identifier) + .join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); + tracing::debug!(?manifest_file, "manifest loaded"); let f = fs::File::open(&manifest_file)?; - Ok(from_reader(f).map_err(RawError::RonError)?) + let manifest: RawManifest = from_reader(f).map_err(RawError::RonError)?; + // verify that the folder name `de_DE` matches the value inside the metadata! + assert_eq!(manifest.metadata.language_identifier, language_identifier); + Ok(manifest) } -/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` files -pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result, common_assets::Error> { +/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, +/// `fr_FR` files +pub(crate) fn load_raw_language( + i18n_root_path: &Path, + manifest: RawManifest, +) -> Result, common_assets::Error> { let language_identifier = &manifest.metadata.language_identifier; - let fragments = recursive_load_raw_language(i18n_root_path, language_identifier, Path::new(""))?; - Ok(RawLanguage{ + let i18n_path = i18n_root_path.join(language_identifier); + + //get List of files + let files = fragments_pathes_in_language(i18n_root_path, language_identifier)?; + + // Walk through each file in the directory + let mut fragments = HashMap::new(); + for fragment_file in &files { + let relative_path = fragment_file.strip_prefix(&i18n_path).unwrap(); + let f = fs::File::open(fragment_file)?; + let fragment = from_reader(f).map_err(RawError::RonError)?; + fragments.insert(relative_path.to_path_buf(), fragment); + } + + Ok(RawLanguage { manifest, fragments, }) } -fn recursive_load_raw_language(i18n_root_path: &Path, language_identifier: &str, subfolder: &Path) -> Result>, common_assets::Error> { - // Walk through each file in the directory - let mut fragments = HashMap::new(); - let search_dir = i18n_root_path.join(language_identifier).join(subfolder); - for fragment_file in search_dir.read_dir().unwrap().flatten() { - let file_type = fragment_file.file_type()?; - if file_type.is_dir() { - let full_path = fragment_file.path(); - let relative_path = full_path.strip_prefix(&search_dir).unwrap(); - fragments.extend(recursive_load_raw_language(i18n_root_path, language_identifier, relative_path)?); - } else if file_type.is_file() { - let full_path = fragment_file.path(); - let relative_path = full_path.strip_prefix(&i18n_root_path).unwrap(); - let f = fs::File::open(&full_path)?; - let fragment = from_reader(f).map_err(RawError::RonError)?; - fragments.insert(relative_path.to_path_buf(), fragment); - } - } - Ok(fragments) -} - impl From> for Language { fn from(raw: RawLanguage) -> Self { - let mut string_map = HashMap::new(); let mut vector_map = HashMap::new(); @@ -105,11 +121,56 @@ impl From> for Language { vector_map, convert_utf8_to_ascii, fonts: raw.manifest.fonts, - metadata: metadata, + metadata, } } } +pub(crate) fn fragments_pathes_in_language( + i18n_root_path: &Path, + language_identifier: &str, +) -> Result, std::io::Error> { + let mut result = vec![]; + recursive_fragments_paths_in_language( + i18n_root_path, + language_identifier, + Path::new(""), + &mut result, + )?; + Ok(result) +} + +/// i18n_path = i18n_root_path.join(REFERENCE_LANG); +fn recursive_fragments_paths_in_language( + i18n_root_path: &Path, + language_identifier: &str, + subfolder: &Path, + result: &mut Vec, +) -> Result<(), std::io::Error> { + let i18n_path = i18n_root_path.join(language_identifier); + let search_dir = i18n_path.join(subfolder); + for fragment_file in search_dir.read_dir().unwrap().flatten() { + let file_type = fragment_file.file_type()?; + if file_type.is_dir() { + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(&i18n_path).unwrap(); + recursive_fragments_paths_in_language( + i18n_root_path, + language_identifier, + relative_path, + result, + )?; + } else if file_type.is_file() { + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(&i18n_path).unwrap(); + if relative_path != Path::new(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)) { + result.push(relative_path.to_path_buf()); + } + } + } + Ok(()) +} + impl core::fmt::Display for RawError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { @@ -120,14 +181,10 @@ impl core::fmt::Display for RawError { impl std::error::Error for RawError {} - impl From for common_assets::Error { - fn from(e: RawError) -> Self { - Self::Conversion(Box::new(e)) - } + fn from(e: RawError) -> Self { Self::Conversion(Box::new(e)) } } - impl common_assets::Asset for RawManifest { type Loader = common_assets::RonLoader; @@ -138,4 +195,4 @@ impl common_assets::Asset for RawFragment { type Loader = common_assets::RonLoader; const EXTENSION: &'static str = LANG_EXTENSION; -} \ No newline at end of file +} diff --git a/voxygen/i18n/src/stats.rs b/voxygen/i18n/src/stats.rs new file mode 100644 index 0000000000..a353e7ef29 --- /dev/null +++ b/voxygen/i18n/src/stats.rs @@ -0,0 +1,205 @@ +use crate::{ + gitfragments::{LocalizationEntryState, LocalizationState, ALL_LOCALIZATION_STATES}, + raw::RawLanguage, +}; +use hashbrown::HashMap; +use std::path::PathBuf; + +#[derive(Default, Debug, PartialEq)] +pub(crate) struct LocalizationStats { + pub(crate) uptodate_entries: usize, + pub(crate) notfound_entries: usize, + pub(crate) unused_entries: usize, + pub(crate) outdated_entries: usize, + pub(crate) errors: usize, +} + +#[allow(clippy::type_complexity)] +pub(crate) struct LocalizationAnalysis { + language_identifier: String, + pub(crate) data: HashMap, Vec<(PathBuf, String, Option)>>, +} + +impl LocalizationStats { + /// Calculate key count that actually matter for the status of the + /// translation Unused entries don't break the game + pub(crate) fn get_real_entry_count(&self) -> usize { + self.outdated_entries + self.notfound_entries + self.errors + self.uptodate_entries + } +} + +impl LocalizationAnalysis { + pub(crate) fn new(language_identifier: &str) -> Self { + let mut data = HashMap::new(); + data.insert(Some(LocalizationState::UpToDate), vec![]); + data.insert(Some(LocalizationState::NotFound), vec![]); + data.insert(Some(LocalizationState::Unused), vec![]); + data.insert(Some(LocalizationState::Outdated), vec![]); + data.insert(None, vec![]); + Self { + language_identifier: language_identifier.to_owned(), + data, + } + } + + fn show( + &self, + state: Option, + reference_language: &RawLanguage, + be_verbose: bool, + ) { + let entries = self.data.get(&state).unwrap_or_else(|| { + panic!( + "called on invalid state: {}", + LocalizationState::print(&state) + ) + }); + if entries.is_empty() { + return; + } + println!("\n\t[{}]", LocalizationState::print(&state)); + for (path, key, commit_id) in entries { + if be_verbose { + let our_commit = LocalizationAnalysis::print_commit(commit_id); + let ref_commit = reference_language + .fragments + .get(path) + .and_then(|entry| entry.string_map.get(key)) + .and_then(|s| s.commit_id) + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()); + println!("{:60}| {:40} | {:40}", key, our_commit, ref_commit,); + } else { + println!("{}", key); + } + } + } + + fn csv(&self, state: Option) { + let entries = self + .data + .get(&state) + .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); + for (path, key, commit_id) in entries { + let our_commit = LocalizationAnalysis::print_commit(commit_id); + println!( + "{},{:?},{},{},{}", + self.language_identifier, + path, + key, + LocalizationState::print(&state), + our_commit + ); + } + } + + fn print_commit(commit_id: &Option) -> String { + commit_id + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()) + } +} + +pub(crate) fn print_translation_stats( + language_identifier: &str, + reference_language: &RawLanguage, + stats: &LocalizationStats, + state_map: &LocalizationAnalysis, + be_verbose: bool, +) { + let real_entry_count = stats.get_real_entry_count() as f32; + let uptodate_percent = (stats.uptodate_entries as f32 / real_entry_count) * 100_f32; + let outdated_percent = (stats.outdated_entries as f32 / real_entry_count) * 100_f32; + let untranslated_percent = ((stats.errors + stats.errors) as f32 / real_entry_count) * 100_f32; + + // Display + if be_verbose { + println!( + "\n{:60}| {:40} | {:40}", + "Key name", + language_identifier, + reference_language.manifest.metadata.language_identifier, + ); + } else { + println!("\nKey name"); + } + + for state in &ALL_LOCALIZATION_STATES { + if state == &Some(LocalizationState::UpToDate) { + continue; + } + state_map.show(*state, reference_language, be_verbose); + } + + println!( + "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", + stats.uptodate_entries, + stats.outdated_entries, + stats.unused_entries, + stats.notfound_entries, + stats.errors, + ); + + println!( + "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", + uptodate_percent, outdated_percent, untranslated_percent, + ); +} + +pub(crate) fn print_csv_file(state_map: &LocalizationAnalysis) { + println!("country_code,file_name,translation_code,status,git_commit"); + + for state in &ALL_LOCALIZATION_STATES { + if state == &Some(LocalizationState::UpToDate) { + continue; + } + state_map.csv(*state); + } +} + +pub(crate) fn print_overall_stats( + analysis: HashMap, +) { + let mut overall_uptodate_entry_count = 0; + let mut overall_outdated_entry_count = 0; + let mut overall_untranslated_entry_count = 0; + let mut overall_real_entry_count = 0; + + println!("-----------------------------------------------------------------------------"); + println!("Overall Translation Status"); + println!("-----------------------------------------------------------------------------"); + println!( + "{:12}| {:8} | {:8} | {:8} | {:8} | {:8}", + "", "up-to-date", "outdated", "untranslated", "unused", "errors", + ); + + let mut i18n_stats: Vec<(&String, &(_, LocalizationStats))> = analysis.iter().collect(); + i18n_stats.sort_by_key(|(_, (_, v))| v.notfound_entries); + + for (path, (_, test_result)) in i18n_stats { + let LocalizationStats { + uptodate_entries: uptodate, + outdated_entries: outdated, + unused_entries: unused, + notfound_entries: untranslated, + errors, + } = test_result; + overall_uptodate_entry_count += uptodate; + overall_outdated_entry_count += outdated; + overall_untranslated_entry_count += untranslated; + overall_real_entry_count += test_result.get_real_entry_count(); + + println!( + "{:12}|{:8} |{:6} |{:8} |{:6} |{:8}", + path, uptodate, outdated, untranslated, unused, errors, + ); + } + + println!( + "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", + (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + ); + println!("-----------------------------------------------------------------------------\n"); +} diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index 89d8146757..6e0c697eab 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -1,23 +1,23 @@ -use std::{path::Path}; +use std::path::Path; -use crate::{i18n_directories, LANG_MANIFEST_FILE, REFERENCE_LANG}; -use crate::raw; +use crate::{i18n_directories, raw, LANG_MANIFEST_FILE, REFERENCE_LANG}; /// Test to verify all languages that they are VALID and loadable, without /// need of git just on the local assets folder -/// `root_dir` - absolute path to main repo -/// `asset_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { - let i18n_root_path = root_dir.join(asset_path); +/// `root_path` - absolute path to main repo +/// `relative_i18n_root_path` - relative path to asset directory (right now it +/// is 'assets/voxygen/i18n') +pub fn verify_all_localizations(root_path: &Path, relative_i18n_root_path: &Path) { + let i18n_root_path = root_path.join(relative_i18n_root_path); let ref_i18n_path = i18n_root_path.join(REFERENCE_LANG); - let ref_i18n_manifest_path = ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION); + let ref_i18n_manifest_path = + ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION); assert!( - root_dir.join(&ref_i18n_path).is_dir(), + root_path.join(&ref_i18n_path).is_dir(), "Reference language folder doesn't exist, something is wrong!" ); assert!( - root_dir.join(&ref_i18n_manifest_path).is_file(), + root_path.join(&ref_i18n_manifest_path).is_file(), "Reference language manifest file doesn't exist, something is wrong!" ); let i18n_directories = i18n_directories(&i18n_root_path); @@ -31,19 +31,29 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { folder is empty?" ); for i18n_directory in i18n_directories { - let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().to_str().unwrap(); - let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().to_str().unwrap(); - println!( - "verifying {:?}", - display_language_identifier - ); + let display_language_identifier = i18n_directory + .strip_prefix(&root_path) + .unwrap() + .to_str() + .unwrap(); + let language_identifier = i18n_directory + .strip_prefix(&i18n_root_path) + .unwrap() + .to_str() + .unwrap(); + println!("verifying {:?}", display_language_identifier); // Walk through each files and try to load them - verify_localization_directory(root_dir, &asset_path, language_identifier); + verify_localization_directory(root_path, relative_i18n_root_path, language_identifier); } } -fn verify_localization_directory(root_dir: &Path, asset_path: &Path, language_identifier: &str) { - let i18n_path = root_dir.join(asset_path); - let manifest = raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file"); +fn verify_localization_directory( + root_path: &Path, + relative_i18n_root_path: &Path, + language_identifier: &str, +) { + let i18n_path = root_path.join(relative_i18n_root_path); + let manifest = + raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file"); raw::load_raw_language(&i18n_path, manifest).expect("error accessing fragment file"); -} \ No newline at end of file +} From c3aa4c54a99eef421ef421e4a11fe5703bdaea9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Thu, 29 Jul 2021 11:29:15 +0200 Subject: [PATCH 5/7] extract path logic --- voxygen/i18n/src/analysis.rs | 148 +++++++++-------------------- voxygen/i18n/src/bin/i18n-check.rs | 20 +--- voxygen/i18n/src/lib.rs | 35 +++---- voxygen/i18n/src/path.rs | 138 +++++++++++++++++++++++++++ voxygen/i18n/src/raw.rs | 100 ++++--------------- voxygen/i18n/src/stats.rs | 12 +-- voxygen/i18n/src/verification.rs | 49 +++------- 7 files changed, 233 insertions(+), 269 deletions(-) create mode 100644 voxygen/i18n/src/path.rs diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index c336816f28..e15d298839 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -2,7 +2,7 @@ use crate::{ gitfragments::{ read_file_from_path, transform_fragment, LocalizationEntryState, LocalizationState, }, - i18n_directories, + path::{BasePath, LangPath}, raw::{self, RawFragment, RawLanguage}, stats::{ print_csv_file, print_overall_stats, print_translation_stats, LocalizationAnalysis, @@ -10,45 +10,35 @@ use crate::{ }, REFERENCE_LANG, }; -use hashbrown::{hash_map::Entry, HashMap, HashSet}; +use hashbrown::{hash_map::Entry, HashMap}; use ron::de::from_bytes; -use std::path::Path; /// Fill the entry State base information (except `state`) for a complete /// language fn gather_entry_state<'a>( repo: &'a git2::Repository, head_ref: &git2::Reference, - language_identifier: &str, - root_path: &Path, - relative_i18n_root_path: &Path, + path: &LangPath, ) -> RawLanguage { - println!("-> {:?}", &language_identifier); - let i18n_root_path = root_path.join(relative_i18n_root_path); + println!("-> {:?}", path.language_identifier()); // load standard manifest - let manifest = raw::load_manifest(&i18n_root_path, language_identifier) - .expect("failed to load language manifest"); + let manifest = raw::load_manifest(path).expect("failed to load language manifest"); // transform language into LocalizationEntryState let mut fragments = HashMap::new(); // For each file in directory - let files = raw::fragments_pathes_in_language(&i18n_root_path, language_identifier) + let files = path + .fragments() .expect("failed to get all files in language"); - for subpath in files { - let path = relative_i18n_root_path - .join(language_identifier) - .join(&subpath); - println!(" -> {:?}", &subpath); - let i18n_blob = read_file_from_path(repo, head_ref, &path); - let fragment: RawFragment = from_bytes(i18n_blob.content()).unwrap_or_else(|e| { - panic!( - "Could not parse {} RON file, skipping: {}", - subpath.to_string_lossy(), - e - ) - }); - let frag = transform_fragment(repo, (&path, fragment), &i18n_blob); - fragments.insert(subpath.to_path_buf(), frag); + for sub_path in files { + let fullpath = path.sub_path(&sub_path); + let gitpath = fullpath.strip_prefix(path.base().root_path()).unwrap(); + println!(" -> {:?}", &sub_path); + let i18n_blob = read_file_from_path(repo, head_ref, gitpath); + let fragment: RawFragment = from_bytes(i18n_blob.content()) + .unwrap_or_else(|e| panic!("Could not parse {:?} RON file, error: {}", sub_path, e)); + let frag = transform_fragment(repo, (gitpath, fragment), &i18n_blob); + fragments.insert(sub_path, frag); } RawLanguage:: { @@ -138,11 +128,10 @@ fn compare_lang_with_reference( } } - let ref_keys: HashSet<&String> = ref_fragment.string_map.keys().collect(); for (_, state) in cur_fragment .string_map .iter_mut() - .filter(|&(k, _)| !ref_keys.contains(k)) + .filter(|&(k, _)| ref_fragment.string_map.get(k).is_none()) { state.state = Some(LocalizationState::Unused); } @@ -179,67 +168,42 @@ fn gather_results( (state_map, stats) } -/// completely analysis multiple languages without printing -fn complete_analysis( - language_identifiers: &[&str], - root_path: &Path, - relative_i18n_root_path: &Path, -) -> ( - HashMap, - /* ref lang */ RawLanguage, -) { - let mut result = HashMap::new(); - // Initialize Git objects - let repo = git2::Repository::discover(&root_path) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_path)); - let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - - // Read Reference Language - let ref_language = gather_entry_state( - &repo, - &head_ref, - REFERENCE_LANG, - root_path, - relative_i18n_root_path, - ); - for &language_identifier in language_identifiers { - let mut cur_language = gather_entry_state( - &repo, - &head_ref, - language_identifier, - root_path, - relative_i18n_root_path, - ); - compare_lang_with_reference(&mut cur_language, &ref_language, &repo); - let (state_map, stats) = gather_results(&cur_language); - result.insert(language_identifier.to_owned(), (state_map, stats)); - } - (result, ref_language) -} - /// Test one language /// - `code`: name of the directory in assets (de_DE for example) -/// - `root_path`: absolute path to main repo -/// - `relative_i18n_root_path`: relative path to asset directory (right now it -/// is 'assets/voxygen/i18n') -/// - be_verbose: print extra info -/// - csv_enabled: generate csv files in target folder +/// - `path`: path to repo +/// - `be_verbose`: print extra info +/// - `csv_enabled`: generate csv files in target folder pub fn test_specific_localizations( + path: &BasePath, language_identifiers: &[&str], - root_path: &Path, - relative_i18n_root_path: &Path, be_verbose: bool, csv_enabled: bool, ) { - let (analysis, reference_language) = - complete_analysis(language_identifiers, root_path, relative_i18n_root_path); + //complete analysis + let mut analysis = HashMap::new(); + // Initialize Git objects + let repo = git2::Repository::discover(path.root_path()) + .unwrap_or_else(|_| panic!("Failed to open the Git repository {:?}", path.root_path())); + let head_ref = repo.head().expect("Impossible to get the HEAD reference"); + + // Read Reference Language + let ref_language = gather_entry_state(&repo, &head_ref, &path.i18n_path(REFERENCE_LANG)); + for &language_identifier in language_identifiers { + let mut cur_language = + gather_entry_state(&repo, &head_ref, &path.i18n_path(language_identifier)); + compare_lang_with_reference(&mut cur_language, &ref_language, &repo); + let (state_map, stats) = gather_results(&cur_language); + analysis.insert(language_identifier.to_owned(), (state_map, stats)); + } + + //printing for (language_identifier, (state_map, stats)) in &analysis { if csv_enabled { print_csv_file(state_map); } else { print_translation_stats( language_identifier, - &reference_language, + &ref_language, stats, state_map, be_verbose, @@ -252,32 +216,12 @@ pub fn test_specific_localizations( } /// Test all localizations -pub fn test_all_localizations( - root_path: &Path, - relative_i18n_root_path: &Path, - be_verbose: bool, - csv_enabled: bool, -) { - let i18n_root_path = root_path.join(relative_i18n_root_path); +pub fn test_all_localizations(path: &BasePath, be_verbose: bool, csv_enabled: bool) { // Compare to other reference files - let language_identifiers = i18n_directories(&i18n_root_path) - .into_iter() - .map(|p| { - p.strip_prefix(&i18n_root_path) - .unwrap() - .to_str() - .unwrap() - .to_owned() - }) + let languages = path.i18n_directories(); + let language_identifiers = languages + .iter() + .map(|s| s.language_identifier()) .collect::>(); - test_specific_localizations( - &language_identifiers - .iter() - .map(|s| s.as_str()) - .collect::>(), - root_path, - relative_i18n_root_path, - be_verbose, - csv_enabled, - ); + test_specific_localizations(path, &language_identifiers, be_verbose, csv_enabled); } diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index 9972850b15..f9062b7b46 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -1,5 +1,4 @@ use clap::{App, Arg}; -use std::path::Path; use veloren_i18n::{analysis, verification}; fn main() { @@ -37,28 +36,17 @@ fn main() { // Generate paths let root_path = common_assets::find_root().expect("Failed to find root of repository"); - let relative_i18n_root_path = Path::new("assets/voxygen/i18n/"); + let path = veloren_i18n::BasePath::new(&root_path); let be_verbose = matches.is_present("verbose"); let csv_enabled = matches.is_present("csv"); if let Some(code) = matches.value_of("CODE") { - analysis::test_specific_localizations( - &[code], - &root_path, - relative_i18n_root_path, - be_verbose, - csv_enabled, - ); + analysis::test_specific_localizations(&path, &[code], be_verbose, csv_enabled); } if matches.is_present("test") { - analysis::test_all_localizations( - &root_path, - relative_i18n_root_path, - be_verbose, - csv_enabled, - ); + analysis::test_all_localizations(&path, be_verbose, csv_enabled); } if matches.is_present("verify") { - verification::verify_all_localizations(&root_path, relative_i18n_root_path); + verification::verify_all_localizations(&path); } } diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 9360823ee1..160a5118c0 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,29 +1,27 @@ #[cfg(any(feature = "bin", test))] pub mod analysis; #[cfg(any(feature = "bin", test))] -pub mod gitfragments; -pub mod raw; +mod gitfragments; +mod path; +mod raw; #[cfg(any(feature = "bin", test))] pub mod stats; pub mod verification; +//reexport +pub use path::BasePath; + +use crate::path::{LANG_EXTENSION, LANG_MANIFEST_FILE}; use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; use hashbrown::{HashMap, HashSet}; use raw::{RawFragment, RawLanguage, RawManifest}; use serde::{Deserialize, Serialize}; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; +use std::{io, path::PathBuf}; use tracing::warn; /// The reference language, aka the more up-to-date localization data. /// Also the default language at first startup. pub const REFERENCE_LANG: &str = "en"; -pub const LANG_MANIFEST_FILE: &str = "_manifest"; - -pub(crate) const LANG_EXTENSION: &str = "ron"; - /// How a language can be described #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct LanguageMetadata { @@ -327,18 +325,9 @@ pub fn list_localizations() -> Vec { LocalizationList::load_expect_cloned("voxygen.i18n").0 } -/// List localization directories as a `PathBuf` vector -pub fn i18n_directories(i18n_dir: &Path) -> Vec { - fs::read_dir(i18n_dir) - .unwrap() - .map(|res| res.map(|e| e.path()).unwrap()) - .filter(|e| e.is_dir()) - .collect() -} - #[cfg(test)] mod tests { - use std::path::Path; + use crate::path::BasePath; // Test that localization list is loaded (not empty) #[test] @@ -358,9 +347,8 @@ mod tests { #[test] fn verify_all_localizations() { // Generate paths - let i18n_root_path = Path::new("assets/voxygen/i18n/"); let root_dir = common_assets::find_root().expect("Failed to discover repository root"); - crate::verification::verify_all_localizations(&root_dir, i18n_root_path); + crate::verification::verify_all_localizations(&BasePath::new(&root_dir)); } // Test to verify all languages and print missing and faulty localisation @@ -368,8 +356,7 @@ mod tests { #[ignore] fn test_all_localizations() { // Generate paths - let i18n_root_path = Path::new("assets/voxygen/i18n/"); let root_dir = common_assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&root_dir, i18n_root_path, true, false); + crate::analysis::test_all_localizations(&BasePath::new(&root_dir), true, false); } } diff --git a/voxygen/i18n/src/path.rs b/voxygen/i18n/src/path.rs new file mode 100644 index 0000000000..a8f961eef5 --- /dev/null +++ b/voxygen/i18n/src/path.rs @@ -0,0 +1,138 @@ +use std::path::{Path, PathBuf}; + +pub(crate) const LANG_MANIFEST_FILE: &str = "_manifest"; +pub(crate) const LANG_EXTENSION: &str = "ron"; + +#[derive(Clone)] +pub struct BasePath { + ///repo part, git main folder + root_path: PathBuf, + ///relative path to i18n path which contains, currently + /// 'assets/voxygen/i18n' + relative_i18n_root_path: PathBuf, + ///i18n_root_folder + cache: PathBuf, +} + +impl BasePath { + pub fn new(root_path: &Path) -> Self { + let relative_i18n_root_path = Path::new("assets/voxygen/i18n").to_path_buf(); + let cache = root_path.join(&relative_i18n_root_path); + assert!( + cache.is_dir(), + "i18n_root_path folder doesn't exist, something is wrong!" + ); + Self { + root_path: root_path.to_path_buf(), + relative_i18n_root_path, + cache, + } + } + + pub fn root_path(&self) -> &Path { &self.root_path } + + pub fn relative_i18n_root_path(&self) -> &Path { &self.relative_i18n_root_path } + + /// absolute path to `relative_i18n_root_path` + pub fn i18n_root_path(&self) -> &Path { &self.cache } + + pub fn i18n_path(&self, language_identifier: &str) -> LangPath { + LangPath::new(self, language_identifier) + } + + /// List localization directories + pub fn i18n_directories(&self) -> Vec { + std::fs::read_dir(&self.cache) + .unwrap() + .map(|res| res.unwrap()) + .filter(|e| e.file_type().unwrap().is_dir()) + .map(|e| LangPath::new(self, e.file_name().to_str().unwrap())) + .collect() + } +} + +impl core::fmt::Debug for BasePath { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", &self.cache) + } +} + +#[derive(Clone)] +pub struct LangPath { + base: BasePath, + /// `en`, `de_DE`, `fr_FR`, etc.. + language_identifier: String, + /// i18n_path + cache: PathBuf, +} + +impl LangPath { + fn new(base: &BasePath, language_identifier: &str) -> Self { + let cache = base.i18n_root_path().join(language_identifier); + if !cache.is_dir() { + panic!("language folder '{}' doesn't exist", language_identifier); + } + Self { + base: base.clone(), + language_identifier: language_identifier.to_owned(), + cache, + } + } + + pub fn base(&self) -> &BasePath { &self.base } + + pub fn language_identifier(&self) -> &str { &self.language_identifier } + + ///absolute path to `i18n_root_path` + `language_identifier` + pub fn i18n_path(&self) -> &Path { &self.cache } + + /// fragment or manifest file, based on a path + pub fn sub_path(&self, sub_path: &Path) -> PathBuf { self.cache.join(sub_path) } + + /// fragment or manifest file, based on a string without extension + pub fn file(&self, name_without_extension: &str) -> PathBuf { + self.cache + .join(format!("{}.{}", name_without_extension, LANG_EXTENSION)) + } + + /// return all fragments sub_pathes + pub(crate) fn fragments(&self) -> Result, std::io::Error> { + let mut result = vec![]; + recursive_fragments_paths_in_language(self, Path::new(""), &mut result)?; + Ok(result) + } +} + +//unwraps cant fail as they are in same Path +fn recursive_fragments_paths_in_language( + lpath: &LangPath, + subfolder: &Path, + result: &mut Vec, +) -> Result<(), std::io::Error> { + let search_dir = lpath.sub_path(subfolder); + for fragment_file in search_dir.read_dir()?.flatten() { + let file_type = fragment_file.file_type()?; + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(lpath.i18n_path()).unwrap(); + if file_type.is_dir() { + recursive_fragments_paths_in_language(lpath, relative_path, result)?; + } else if file_type.is_file() + && relative_path != Path::new(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)) + { + result.push(relative_path.to_path_buf()); + } + } + Ok(()) +} + +impl core::fmt::Debug for LangPath { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{:?}", + self.base + .relative_i18n_root_path + .join(&self.language_identifier) + ) + } +} diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs index e3e00a8d4e..56e4e4b1c7 100644 --- a/voxygen/i18n/src/raw.rs +++ b/voxygen/i18n/src/raw.rs @@ -1,24 +1,13 @@ //! handle the loading of a `Language` -//! Paths: -//! - `root_path`: repo part, git main folder -//! - `language_identifier`: `en`, `de_DE`, `fr_FR`, etc.. -//! - `relative_i18n_root_path`: relative path to i18n path which contains -//! `language_identifier` folders from `root_path` -//! - `i18n_root_path`: absolute path to `relative_i18n_root_path` -//! - `i18n_path`: absolute path to `i18n_root_path` + `language_identifier` -//! - `subfolder`: all folders in `i18n_path` -//! -//! wherever possible we use relative paths only. So expect 1 absolute -//! `root_path` or `i18n_root_path` to be required and all others be relative. -use crate::{Fonts, Language, LanguageMetadata, LANG_EXTENSION, LANG_MANIFEST_FILE}; +use crate::{ + path::{LangPath, LANG_EXTENSION, LANG_MANIFEST_FILE}, + Fonts, Language, LanguageMetadata, +}; use deunicode::deunicode; use hashbrown::hash_map::HashMap; use ron::de::from_reader; use serde::{Deserialize, Serialize}; -use std::{ - fs, - path::{Path, PathBuf}, -}; +use std::{fs, path::PathBuf}; /// Raw localization metadata from LANG_MANIFEST_FILE file /// See `Language` for more info on each attributes @@ -47,42 +36,32 @@ pub(crate) enum RawError { RonError(ron::Error), } -/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, -/// `fr_FR` folders -pub(crate) fn load_manifest( - i18n_root_path: &Path, - language_identifier: &str, -) -> Result { - let manifest_file = i18n_root_path - .join(language_identifier) - .join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); - tracing::debug!(?manifest_file, "manifest loaded"); +pub(crate) fn load_manifest(path: &LangPath) -> Result { + let manifest_file = path.file(LANG_MANIFEST_FILE); + tracing::debug!(?manifest_file, "manifest loading"); let f = fs::File::open(&manifest_file)?; let manifest: RawManifest = from_reader(f).map_err(RawError::RonError)?; // verify that the folder name `de_DE` matches the value inside the metadata! - assert_eq!(manifest.metadata.language_identifier, language_identifier); + assert_eq!( + manifest.metadata.language_identifier, + path.language_identifier() + ); Ok(manifest) } -/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, -/// `fr_FR` files pub(crate) fn load_raw_language( - i18n_root_path: &Path, + path: &LangPath, manifest: RawManifest, ) -> Result, common_assets::Error> { - let language_identifier = &manifest.metadata.language_identifier; - let i18n_path = i18n_root_path.join(language_identifier); - //get List of files - let files = fragments_pathes_in_language(i18n_root_path, language_identifier)?; + let files = path.fragments()?; // Walk through each file in the directory let mut fragments = HashMap::new(); - for fragment_file in &files { - let relative_path = fragment_file.strip_prefix(&i18n_path).unwrap(); - let f = fs::File::open(fragment_file)?; + for sub_path in files { + let f = fs::File::open(path.sub_path(&sub_path))?; let fragment = from_reader(f).map_err(RawError::RonError)?; - fragments.insert(relative_path.to_path_buf(), fragment); + fragments.insert(sub_path, fragment); } Ok(RawLanguage { @@ -126,51 +105,6 @@ impl From> for Language { } } -pub(crate) fn fragments_pathes_in_language( - i18n_root_path: &Path, - language_identifier: &str, -) -> Result, std::io::Error> { - let mut result = vec![]; - recursive_fragments_paths_in_language( - i18n_root_path, - language_identifier, - Path::new(""), - &mut result, - )?; - Ok(result) -} - -/// i18n_path = i18n_root_path.join(REFERENCE_LANG); -fn recursive_fragments_paths_in_language( - i18n_root_path: &Path, - language_identifier: &str, - subfolder: &Path, - result: &mut Vec, -) -> Result<(), std::io::Error> { - let i18n_path = i18n_root_path.join(language_identifier); - let search_dir = i18n_path.join(subfolder); - for fragment_file in search_dir.read_dir().unwrap().flatten() { - let file_type = fragment_file.file_type()?; - if file_type.is_dir() { - let full_path = fragment_file.path(); - let relative_path = full_path.strip_prefix(&i18n_path).unwrap(); - recursive_fragments_paths_in_language( - i18n_root_path, - language_identifier, - relative_path, - result, - )?; - } else if file_type.is_file() { - let full_path = fragment_file.path(); - let relative_path = full_path.strip_prefix(&i18n_path).unwrap(); - if relative_path != Path::new(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)) { - result.push(relative_path.to_path_buf()); - } - } - } - Ok(()) -} - impl core::fmt::Display for RawError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { diff --git a/voxygen/i18n/src/stats.rs b/voxygen/i18n/src/stats.rs index a353e7ef29..9ab93fd062 100644 --- a/voxygen/i18n/src/stats.rs +++ b/voxygen/i18n/src/stats.rs @@ -45,7 +45,7 @@ impl LocalizationAnalysis { fn show( &self, state: Option, - reference_language: &RawLanguage, + ref_language: &RawLanguage, be_verbose: bool, ) { let entries = self.data.get(&state).unwrap_or_else(|| { @@ -61,7 +61,7 @@ impl LocalizationAnalysis { for (path, key, commit_id) in entries { if be_verbose { let our_commit = LocalizationAnalysis::print_commit(commit_id); - let ref_commit = reference_language + let ref_commit = ref_language .fragments .get(path) .and_then(|entry| entry.string_map.get(key)) @@ -102,7 +102,7 @@ impl LocalizationAnalysis { pub(crate) fn print_translation_stats( language_identifier: &str, - reference_language: &RawLanguage, + ref_language: &RawLanguage, stats: &LocalizationStats, state_map: &LocalizationAnalysis, be_verbose: bool, @@ -116,9 +116,7 @@ pub(crate) fn print_translation_stats( if be_verbose { println!( "\n{:60}| {:40} | {:40}", - "Key name", - language_identifier, - reference_language.manifest.metadata.language_identifier, + "Key name", language_identifier, ref_language.manifest.metadata.language_identifier, ); } else { println!("\nKey name"); @@ -128,7 +126,7 @@ pub(crate) fn print_translation_stats( if state == &Some(LocalizationState::UpToDate) { continue; } - state_map.show(*state, reference_language, be_verbose); + state_map.show(*state, ref_language, be_verbose); } println!( diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index 6e0c697eab..fa784b9db5 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -1,26 +1,17 @@ -use std::path::Path; +use crate::path::{BasePath, LangPath, LANG_MANIFEST_FILE}; -use crate::{i18n_directories, raw, LANG_MANIFEST_FILE, REFERENCE_LANG}; +use crate::{raw, REFERENCE_LANG}; /// Test to verify all languages that they are VALID and loadable, without /// need of git just on the local assets folder -/// `root_path` - absolute path to main repo -/// `relative_i18n_root_path` - relative path to asset directory (right now it -/// is 'assets/voxygen/i18n') -pub fn verify_all_localizations(root_path: &Path, relative_i18n_root_path: &Path) { - let i18n_root_path = root_path.join(relative_i18n_root_path); - let ref_i18n_path = i18n_root_path.join(REFERENCE_LANG); - let ref_i18n_manifest_path = - ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION); +pub fn verify_all_localizations(path: &BasePath) { + let ref_i18n_path = path.i18n_path(REFERENCE_LANG); + let ref_i18n_manifest_path = ref_i18n_path.file(LANG_MANIFEST_FILE); assert!( - root_path.join(&ref_i18n_path).is_dir(), - "Reference language folder doesn't exist, something is wrong!" - ); - assert!( - root_path.join(&ref_i18n_manifest_path).is_file(), + ref_i18n_manifest_path.is_file(), "Reference language manifest file doesn't exist, something is wrong!" ); - let i18n_directories = i18n_directories(&i18n_root_path); + let i18n_directories = path.i18n_directories(); // This simple check ONLY guarantees that an arbitrary minimum of translation // files exists. It's just to notice unintentional deletion of all // files, or modifying the paths. In case you want to delete all @@ -31,29 +22,13 @@ pub fn verify_all_localizations(root_path: &Path, relative_i18n_root_path: &Path folder is empty?" ); for i18n_directory in i18n_directories { - let display_language_identifier = i18n_directory - .strip_prefix(&root_path) - .unwrap() - .to_str() - .unwrap(); - let language_identifier = i18n_directory - .strip_prefix(&i18n_root_path) - .unwrap() - .to_str() - .unwrap(); - println!("verifying {:?}", display_language_identifier); + println!("verifying {:?}", i18n_directory); // Walk through each files and try to load them - verify_localization_directory(root_path, relative_i18n_root_path, language_identifier); + verify_localization_directory(&i18n_directory); } } -fn verify_localization_directory( - root_path: &Path, - relative_i18n_root_path: &Path, - language_identifier: &str, -) { - let i18n_path = root_path.join(relative_i18n_root_path); - let manifest = - raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file"); - raw::load_raw_language(&i18n_path, manifest).expect("error accessing fragment file"); +fn verify_localization_directory(path: &LangPath) { + let manifest = raw::load_manifest(path).expect("error accessing manifest file"); + raw::load_raw_language(path, manifest).expect("error accessing fragment file"); } From 4c0b74150df45a6015d31a9da58d627d156b6198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Thu, 29 Jul 2021 20:47:45 +0200 Subject: [PATCH 6/7] remove some reexports --- .gitlab/scripts/unittest.sh | 2 +- Cargo.lock | 30 ++++++++++---------- voxygen/Cargo.toml | 2 +- voxygen/i18n/Cargo.toml | 2 +- voxygen/i18n/src/bin/i18n-check.rs | 4 +-- voxygen/src/hud/bag.rs | 2 +- voxygen/src/hud/buffs.rs | 2 +- voxygen/src/hud/buttons.rs | 2 +- voxygen/src/hud/chat.rs | 3 +- voxygen/src/hud/crafting.rs | 10 +++---- voxygen/src/hud/diary.rs | 6 ++-- voxygen/src/hud/esc_menu.rs | 3 +- voxygen/src/hud/group.rs | 2 +- voxygen/src/hud/loot_scroller.rs | 6 ++-- voxygen/src/hud/map.rs | 2 +- voxygen/src/hud/mod.rs | 2 +- voxygen/src/hud/overhead.rs | 2 +- voxygen/src/hud/overitem.rs | 2 +- voxygen/src/hud/popup.rs | 3 +- voxygen/src/hud/prompt_dialog.rs | 2 +- voxygen/src/hud/settings_window/chat.rs | 2 +- voxygen/src/hud/settings_window/controls.rs | 2 +- voxygen/src/hud/settings_window/gameplay.rs | 2 +- voxygen/src/hud/settings_window/interface.rs | 2 +- voxygen/src/hud/settings_window/language.rs | 2 +- voxygen/src/hud/settings_window/mod.rs | 2 +- voxygen/src/hud/settings_window/sound.rs | 2 +- voxygen/src/hud/settings_window/video.rs | 2 +- voxygen/src/hud/skillbar.rs | 2 +- voxygen/src/hud/social.rs | 7 ++--- voxygen/src/hud/trade.rs | 2 +- voxygen/src/hud/util.rs | 3 +- voxygen/src/lib.rs | 4 --- voxygen/src/main.rs | 2 +- voxygen/src/menu/char_selection/ui/mod.rs | 2 +- voxygen/src/menu/main/mod.rs | 2 +- voxygen/src/menu/main/ui/connecting.rs | 13 ++++----- voxygen/src/menu/main/ui/disclaimer.rs | 2 +- voxygen/src/menu/main/ui/login.rs | 26 ++++++++--------- voxygen/src/menu/main/ui/mod.rs | 10 +++---- voxygen/src/menu/main/ui/servers.rs | 10 +++---- voxygen/src/session/mod.rs | 3 +- voxygen/src/session/settings_change.rs | 2 +- voxygen/src/settings/language.rs | 1 - voxygen/src/ui/cache.rs | 2 +- voxygen/src/ui/fonts.rs | 2 +- voxygen/src/ui/ice/cache.rs | 2 +- voxygen/src/ui/ice/mod.rs | 2 +- voxygen/src/ui/ice/renderer/mod.rs | 2 +- voxygen/src/ui/mod.rs | 2 +- voxygen/src/ui/widgets/item_tooltip.rs | 14 ++++----- voxygen/src/window.rs | 3 +- 52 files changed, 104 insertions(+), 121 deletions(-) diff --git a/.gitlab/scripts/unittest.sh b/.gitlab/scripts/unittest.sh index a4193d3914..4bb46b4995 100755 --- a/.gitlab/scripts/unittest.sh +++ b/.gitlab/scripts/unittest.sh @@ -1,7 +1,7 @@ #!/bin/bash export VELOREN_ASSETS="$(pwd)/assets" rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly -time cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored && +time cargo test --package veloren-voxygen-i18n --lib test_all_localizations -- --nocapture --ignored && time cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib && ( rm -r target/debug/incremental* || echo "all good" ) && # TMP FIX FOR 2021-03-22-nightly time cargo test \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0274d8ca78..3132f6e3fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6017,20 +6017,6 @@ dependencies = [ "veloren-common-net", ] -[[package]] -name = "veloren-i18n" -version = "0.10.0" -dependencies = [ - "clap", - "deunicode", - "git2", - "hashbrown 0.11.2", - "ron", - "serde", - "tracing", - "veloren-common-assets", -] - [[package]] name = "veloren-network" version = "0.3.0" @@ -6236,10 +6222,10 @@ dependencies = [ "veloren-common-net", "veloren-common-state", "veloren-common-systems", - "veloren-i18n", "veloren-server", "veloren-voxygen-anim", "veloren-voxygen-egui", + "veloren-voxygen-i18n", "veloren-world", "wgpu", "wgpu-profiler", @@ -6295,6 +6281,20 @@ dependencies = [ "veloren-voxygen-egui", ] +[[package]] +name = "veloren-voxygen-i18n" +version = "0.10.0" +dependencies = [ + "clap", + "deunicode", + "git2", + "hashbrown 0.11.2", + "ron", + "serde", + "tracing", + "veloren-common-assets", +] + [[package]] name = "veloren-world" version = "0.10.0" diff --git a/voxygen/Cargo.toml b/voxygen/Cargo.toml index 924be04100..c7504d5890 100644 --- a/voxygen/Cargo.toml +++ b/voxygen/Cargo.toml @@ -46,7 +46,7 @@ common-systems = {package = "veloren-common-systems", path = "../common/systems" common-state = {package = "veloren-common-state", path = "../common/state"} anim = {package = "veloren-voxygen-anim", path = "anim"} -i18n = {package = "veloren-i18n", path = "i18n"} +i18n = {package = "veloren-voxygen-i18n", path = "i18n"} voxygen-egui = {package = "veloren-voxygen-egui", path = "egui", optional = true } # Graphics diff --git a/voxygen/i18n/Cargo.toml b/voxygen/i18n/Cargo.toml index 388586679b..5ebbbb05d1 100644 --- a/voxygen/i18n/Cargo.toml +++ b/voxygen/i18n/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["juliancoffee ", "Rémy Phelipot"] edition = "2018" -name = "veloren-i18n" +name = "veloren-voxygen-i18n" description = "Crate for internalization and diagnostic of existing localizations." version = "0.10.0" diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index f9062b7b46..6a868a950f 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -1,5 +1,5 @@ use clap::{App, Arg}; -use veloren_i18n::{analysis, verification}; +use veloren_voxygen_i18n::{analysis, verification, BasePath}; fn main() { let matches = App::new("i18n-check") @@ -36,7 +36,7 @@ fn main() { // Generate paths let root_path = common_assets::find_root().expect("Failed to find root of repository"); - let path = veloren_i18n::BasePath::new(&root_path); + let path = BasePath::new(&root_path); let be_verbose = matches.is_present("verbose"); let csv_enabled = matches.is_present("csv"); diff --git a/voxygen/src/hud/bag.rs b/voxygen/src/hud/bag.rs index 343df2f232..4a21b416db 100644 --- a/voxygen/src/hud/bag.rs +++ b/voxygen/src/hud/bag.rs @@ -7,7 +7,6 @@ use super::{ }; use crate::{ game_input::GameInput, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -31,6 +30,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, State as ConrodState, Text}, widget_ids, Color, Colorable, Positionable, Scalar, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use crate::hud::slots::SlotKind; use specs::Entity as EcsEntity; diff --git a/voxygen/src/hud/buffs.rs b/voxygen/src/hud/buffs.rs index 5fc82242df..393dd06fc0 100644 --- a/voxygen/src/hud/buffs.rs +++ b/voxygen/src/hud/buffs.rs @@ -4,10 +4,10 @@ use super::{ }; use crate::{ hud::{self, BuffPosition}, - i18n::Localization, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, }; +use i18n::Localization; use common::comp::{BuffKind, Buffs, Energy, Health}; use conrod_core::{ diff --git a/voxygen/src/hud/buttons.rs b/voxygen/src/hud/buttons.rs index 79bde0effd..d046c8a783 100644 --- a/voxygen/src/hud/buttons.rs +++ b/voxygen/src/hud/buttons.rs @@ -4,7 +4,6 @@ use super::{ }; use crate::{ game_input::GameInput, - i18n::Localization, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, window::KeyMouse, GlobalState, @@ -15,6 +14,7 @@ use conrod_core::{ widget::{self, Button, Image, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { bag, diff --git a/voxygen/src/hud/chat.rs b/voxygen/src/hud/chat.rs index c05177e1ca..8a48c212e7 100644 --- a/voxygen/src/hud/chat.rs +++ b/voxygen/src/hud/chat.rs @@ -2,7 +2,7 @@ use super::{ img_ids::Imgs, ChatTab, ERROR_COLOR, FACTION_COLOR, GROUP_COLOR, INFO_COLOR, KILL_COLOR, OFFLINE_COLOR, ONLINE_COLOR, REGION_COLOR, SAY_COLOR, TELL_COLOR, TEXT_COLOR, WORLD_COLOR, }; -use crate::{i18n::Localization, settings::chat::MAX_CHAT_TABS, ui::fonts::Fonts, GlobalState}; +use crate::{settings::chat::MAX_CHAT_TABS, ui::fonts::Fonts, GlobalState}; use client::{cmd, Client}; use common::comp::{ chat::{KillSource, KillType}, @@ -22,6 +22,7 @@ use conrod_core::{ widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Ui, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use std::collections::{HashSet, VecDeque}; widget_ids! { diff --git a/voxygen/src/hud/crafting.rs b/voxygen/src/hud/crafting.rs index 79093537ac..dd2e917b9c 100644 --- a/voxygen/src/hud/crafting.rs +++ b/voxygen/src/hud/crafting.rs @@ -4,12 +4,9 @@ use super::{ item_imgs::{animate_by_pulse, ItemImgs, ItemKey::Tool}, Show, TEXT_COLOR, TEXT_DULL_RED_COLOR, TEXT_GRAY_COLOR, UI_HIGHLIGHT_0, UI_MAIN, }; -use crate::{ - i18n::Localization, - ui::{ - fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable, Tooltip, - TooltipManager, Tooltipable, - }, +use crate::ui::{ + fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable, Tooltip, + TooltipManager, Tooltipable, }; use client::{self, Client}; use common::{ @@ -29,6 +26,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text, TextEdit}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::sync::Arc; use strum::IntoEnumIterator; diff --git a/voxygen/src/hud/diary.rs b/voxygen/src/hud/diary.rs index 8112d784d1..da5af98b50 100644 --- a/voxygen/src/hud/diary.rs +++ b/voxygen/src/hud/diary.rs @@ -3,16 +3,14 @@ use super::{ item_imgs::{animate_by_pulse, ItemImgs, ItemKey::Tool}, Show, CRITICAL_HP_COLOR, HP_COLOR, TEXT_COLOR, UI_HIGHLIGHT_0, UI_MAIN, XP_COLOR, }; -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}; use conrod_core::{ color, image::Id, widget::{self, button, Button, Image, Rectangle, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use client::{self, Client}; use common::comp::{ diff --git a/voxygen/src/hud/esc_menu.rs b/voxygen/src/hud/esc_menu.rs index 5573cf3ad6..92f989ad68 100644 --- a/voxygen/src/hud/esc_menu.rs +++ b/voxygen/src/hud/esc_menu.rs @@ -1,9 +1,10 @@ use super::{img_ids::Imgs, settings_window::SettingsTab, TEXT_COLOR}; -use crate::{i18n::Localization, ui::fonts::Fonts}; +use crate::ui::fonts::Fonts; use conrod_core::{ widget::{self, Button, Image}, widget_ids, Color, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/group.rs b/voxygen/src/hud/group.rs index 74ee3ed8b2..71fb4a5f56 100644 --- a/voxygen/src/hud/group.rs +++ b/voxygen/src/hud/group.rs @@ -9,7 +9,6 @@ use super::{ use crate::{ game_input::GameInput, hud, - i18n::Localization, settings::Settings, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, @@ -27,6 +26,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use specs::{saveload::MarkerAllocator, WorldExt}; widget_ids! { diff --git a/voxygen/src/hud/loot_scroller.rs b/voxygen/src/hud/loot_scroller.rs index 676291465b..53e88d6820 100644 --- a/voxygen/src/hud/loot_scroller.rs +++ b/voxygen/src/hud/loot_scroller.rs @@ -4,10 +4,7 @@ use super::{ item_imgs::ItemImgs, Show, Windows, TEXT_COLOR, }; -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable}; use client::Client; use common::comp::inventory::item::{ItemDef, MaterialStatManifest, Quality}; use conrod_core::{ @@ -16,6 +13,7 @@ use conrod_core::{ widget::{self, Image, List, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::{collections::VecDeque, sync::Arc}; widget_ids! { diff --git a/voxygen/src/hud/map.rs b/voxygen/src/hud/map.rs index 3742f2565c..9039add9b4 100644 --- a/voxygen/src/hud/map.rs +++ b/voxygen/src/hud/map.rs @@ -4,7 +4,6 @@ use super::{ TEXT_BG, TEXT_BLUE_COLOR, TEXT_COLOR, TEXT_GRAY_COLOR, TEXT_VELORITE, UI_HIGHLIGHT_0, UI_MAIN, }; use crate::{ - i18n::Localization, session::settings_change::{Interface as InterfaceChange, Interface::*}, ui::{fonts::Fonts, img_ids, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, @@ -17,6 +16,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use specs::{saveload::MarkerAllocator, WorldExt}; use vek::*; diff --git a/voxygen/src/hud/mod.rs b/voxygen/src/hud/mod.rs index af9304279d..4a3b4693ba 100644 --- a/voxygen/src/hud/mod.rs +++ b/voxygen/src/hud/mod.rs @@ -55,7 +55,6 @@ use crate::{ ecs::{comp as vcomp, comp::HpFloaterList}, game_input::GameInput, hud::{img_ids::ImgsRot, prompt_dialog::DialogOutcomeEvent}, - i18n::Localization, render::UiDrawer, scene::camera::{self, Camera}, session::{ @@ -100,6 +99,7 @@ use conrod_core::{ widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, }; use hashbrown::{HashMap, HashSet}; +use i18n::Localization; use rand::Rng; use specs::{Entity as EcsEntity, Join, WorldExt}; use std::{ diff --git a/voxygen/src/hud/overhead.rs b/voxygen/src/hud/overhead.rs index 4d58d7aef0..c0c191e50d 100644 --- a/voxygen/src/hud/overhead.rs +++ b/voxygen/src/hud/overhead.rs @@ -5,7 +5,6 @@ use super::{ }; use crate::{ hud::{get_buff_image, get_buff_info}, - i18n::Localization, settings::InterfaceSettings, ui::{fonts::Fonts, Ingameable}, }; @@ -16,6 +15,7 @@ use conrod_core::{ widget::{self, Image, Rectangle, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; const MAX_BUBBLE_WIDTH: f64 = 250.0; widget_ids! { diff --git a/voxygen/src/hud/overitem.rs b/voxygen/src/hud/overitem.rs index 368350389b..7522e04f06 100644 --- a/voxygen/src/hud/overitem.rs +++ b/voxygen/src/hud/overitem.rs @@ -1,6 +1,5 @@ use crate::{ game_input::GameInput, - i18n::Localization, settings::ControlSettings, ui::{fonts::Fonts, Ingameable}, }; @@ -9,6 +8,7 @@ use conrod_core::{ widget::{self, RoundedRectangle, Text}, widget_ids, Color, Colorable, Positionable, Widget, WidgetCommon, }; +use i18n::Localization; use std::borrow::Cow; use keyboard_keynames::key_layout::KeyLayout; diff --git a/voxygen/src/hud/popup.rs b/voxygen/src/hud/popup.rs index 843bc57baf..a87feaecb2 100644 --- a/voxygen/src/hud/popup.rs +++ b/voxygen/src/hud/popup.rs @@ -1,11 +1,12 @@ use super::Show; -use crate::{i18n::Localization, ui::fonts::Fonts}; +use crate::ui::fonts::Fonts; use client::{self, Client}; use common_net::msg::Notification; use conrod_core::{ widget::{self, Text}, widget_ids, Color, Colorable, Positionable, Widget, WidgetCommon, }; +use i18n::Localization; use std::{collections::VecDeque, time::Instant}; widget_ids! { diff --git a/voxygen/src/hud/prompt_dialog.rs b/voxygen/src/hud/prompt_dialog.rs index 599eca1953..769100fc86 100644 --- a/voxygen/src/hud/prompt_dialog.rs +++ b/voxygen/src/hud/prompt_dialog.rs @@ -2,7 +2,6 @@ use super::{img_ids::Imgs, TEXT_COLOR, UI_HIGHLIGHT_0}; use crate::{ game_input::GameInput, hud::{Event, PromptDialogSettings}, - i18n::LocalizationHandle, settings::Settings, ui::fonts::Fonts, }; @@ -10,6 +9,7 @@ use conrod_core::{ widget::{self, Button, Image, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::LocalizationHandle; use keyboard_keynames::key_layout::KeyLayout; widget_ids! { diff --git a/voxygen/src/hud/settings_window/chat.rs b/voxygen/src/hud/settings_window/chat.rs index 60c540d8fd..6f74fcd20b 100644 --- a/voxygen/src/hud/settings_window/chat.rs +++ b/voxygen/src/hud/settings_window/chat.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, ChatTab, Show, TEXT_COLOR, TEXT_GRAY_COLOR, UI_HIGHLIGHT_0, UI_MAIN}, - i18n::Localization, session::settings_change::{Chat as ChatChange, Chat::*}, settings::chat::MAX_CHAT_TABS, ui::{fonts::Fonts, ImageSlider, ToggleButton}, @@ -14,6 +13,7 @@ use conrod_core::{ widget::{self, Button, DropDownList, Image, Rectangle, Text, TextEdit}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::cmp::Ordering; widget_ids! { diff --git a/voxygen/src/hud/settings_window/controls.rs b/voxygen/src/hud/settings_window/controls.rs index 51b842f936..005ba006b6 100644 --- a/voxygen/src/hud/settings_window/controls.rs +++ b/voxygen/src/hud/settings_window/controls.rs @@ -3,7 +3,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ game_input::GameInput, hud::{img_ids::Imgs, ERROR_COLOR, TEXT_BIND_CONFLICT_COLOR, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Control as ControlChange, Control::*}, ui::fonts::Fonts, GlobalState, @@ -14,6 +13,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Borderable, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use strum::IntoEnumIterator; widget_ids! { diff --git a/voxygen/src/hud/settings_window/gameplay.rs b/voxygen/src/hud/settings_window/gameplay.rs index bb059f528d..010c71739b 100644 --- a/voxygen/src/hud/settings_window/gameplay.rs +++ b/voxygen/src/hud/settings_window/gameplay.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, PressBehavior, MENU_BG, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Gameplay as GameplayChange, Gameplay::*}, ui::{fonts::Fonts, ImageSlider, ToggleButton}, GlobalState, @@ -13,6 +12,7 @@ use conrod_core::{ widget::{self, Button, DropDownList, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/interface.rs b/voxygen/src/hud/settings_window/interface.rs index f39def8fa6..27f4745a60 100644 --- a/voxygen/src/hud/settings_window/interface.rs +++ b/voxygen/src/hud/settings_window/interface.rs @@ -4,7 +4,6 @@ use crate::{ hud::{ img_ids::Imgs, BarNumbers, BuffPosition, CrosshairType, ShortcutNumbers, Show, TEXT_COLOR, }, - i18n::Localization, session::settings_change::{Interface as InterfaceChange, Interface::*}, ui::{fonts::Fonts, ImageSlider, ScaleMode, ToggleButton}, GlobalState, @@ -15,6 +14,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids{ diff --git a/voxygen/src/hud/settings_window/language.rs b/voxygen/src/hud/settings_window/language.rs index e1cb5787c7..5745b81bca 100644 --- a/voxygen/src/hud/settings_window/language.rs +++ b/voxygen/src/hud/settings_window/language.rs @@ -1,6 +1,5 @@ use crate::{ hud::{img_ids::Imgs, TEXT_COLOR}, - i18n::{list_localizations, Localization}, session::settings_change::{Language as LanguageChange, Language::*}, ui::{fonts::Fonts, ToggleButton}, GlobalState, @@ -10,6 +9,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::{list_localizations, Localization}; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/mod.rs b/voxygen/src/hud/settings_window/mod.rs index 73de93697a..e72101755f 100644 --- a/voxygen/src/hud/settings_window/mod.rs +++ b/voxygen/src/hud/settings_window/mod.rs @@ -8,7 +8,6 @@ mod video; use crate::{ hud::{img_ids::Imgs, Show, TEXT_COLOR, UI_HIGHLIGHT_0, UI_MAIN}, - i18n::Localization, session::settings_change::SettingsChange, ui::fonts::Fonts, GlobalState, @@ -18,6 +17,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use strum::IntoEnumIterator; use strum_macros::EnumIter; diff --git a/voxygen/src/hud/settings_window/sound.rs b/voxygen/src/hud/settings_window/sound.rs index f563b48f9e..ed655a0765 100644 --- a/voxygen/src/hud/settings_window/sound.rs +++ b/voxygen/src/hud/settings_window/sound.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Audio as AudioChange, Audio::*}, ui::{fonts::Fonts, ImageSlider}, GlobalState, @@ -13,6 +12,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/video.rs b/voxygen/src/hud/settings_window/video.rs index c638602f30..e577ecb6d6 100644 --- a/voxygen/src/hud/settings_window/video.rs +++ b/voxygen/src/hud/settings_window/video.rs @@ -5,7 +5,6 @@ use crate::{ img_ids::Imgs, CRITICAL_HP_COLOR, HP_COLOR, LOW_HP_COLOR, MENU_BG, STAMINA_COLOR, TEXT_COLOR, }, - i18n::Localization, render::{ AaMode, CloudMode, FluidMode, LightingMode, PresentMode, RenderMode, ShadowMapMode, ShadowMode, UpscaleMode, @@ -23,6 +22,7 @@ use conrod_core::{ widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; use core::convert::TryFrom; +use i18n::Localization; use itertools::Itertools; use std::iter::once; diff --git a/voxygen/src/hud/skillbar.rs b/voxygen/src/hud/skillbar.rs index debac354c9..07eae60057 100644 --- a/voxygen/src/hud/skillbar.rs +++ b/voxygen/src/hud/skillbar.rs @@ -8,7 +8,6 @@ use super::{ use crate::{ game_input::GameInput, hud::{ComboFloater, Position, PositionSpecifier}, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -17,6 +16,7 @@ use crate::{ }, GlobalState, }; +use i18n::Localization; use client::{self, Client}; use common::comp::{ diff --git a/voxygen/src/hud/social.rs b/voxygen/src/hud/social.rs index 8c81cf06a7..6af6a1494e 100644 --- a/voxygen/src/hud/social.rs +++ b/voxygen/src/hud/social.rs @@ -2,11 +2,7 @@ use super::{ img_ids::{Imgs, ImgsRot}, Show, TEXT_COLOR, TEXT_COLOR_3, UI_HIGHLIGHT_0, UI_MAIN, }; - -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}; use client::{self, Client}; use common::{comp::group, uid::Uid}; use conrod_core::{ @@ -14,6 +10,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text, TextEdit}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use itertools::Itertools; use std::time::Instant; diff --git a/voxygen/src/hud/trade.rs b/voxygen/src/hud/trade.rs index 1aa46459e9..83023f4e7d 100644 --- a/voxygen/src/hud/trade.rs +++ b/voxygen/src/hud/trade.rs @@ -6,7 +6,6 @@ use super::{ }; use crate::{ hud::bag::{BackgroundIds, InventoryScroller}, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -28,6 +27,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, State as ConrodState, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use specs::Entity as EcsEntity; use vek::*; diff --git a/voxygen/src/hud/util.rs b/voxygen/src/hud/util.rs index e383027404..0b7342abf1 100644 --- a/voxygen/src/hud/util.rs +++ b/voxygen/src/hud/util.rs @@ -11,10 +11,9 @@ use common::{ effect::Effect, trade::{Good, SitePrices}, }; +use i18n::Localization; use std::{borrow::Cow, fmt::Write}; -use crate::i18n::Localization; - pub fn price_desc( prices: &Option, item_definition_id: &str, diff --git a/voxygen/src/lib.rs b/voxygen/src/lib.rs index 304710c011..cf4d673534 100644 --- a/voxygen/src/lib.rs +++ b/voxygen/src/lib.rs @@ -33,10 +33,6 @@ pub mod settings; pub mod singleplayer; pub mod window; -// Reexports -pub use crate::error::Error; -pub use i18n; - #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; #[cfg(feature = "egui-ui")] diff --git a/voxygen/src/main.rs b/voxygen/src/main.rs index 2a52b98e68..f7dff8f1c7 100644 --- a/voxygen/src/main.rs +++ b/voxygen/src/main.rs @@ -2,9 +2,9 @@ #![feature(bool_to_option)] #![recursion_limit = "2048"] +use i18n::{self, LocalizationHandle}; use veloren_voxygen::{ audio::AudioFrontend, - i18n::{self, LocalizationHandle}, profile::Profile, run, scene::terrain::SpriteRenderContext, diff --git a/voxygen/src/menu/char_selection/ui/mod.rs b/voxygen/src/menu/char_selection/ui/mod.rs index d48bfafe6e..b26d7c8fbe 100644 --- a/voxygen/src/menu/char_selection/ui/mod.rs +++ b/voxygen/src/menu/char_selection/ui/mod.rs @@ -1,5 +1,4 @@ use crate::{ - i18n::{Localization, LocalizationHandle}, render::UiDrawer, ui::{ self, @@ -26,6 +25,7 @@ use common::{ comp::{self, humanoid, inventory::slot::EquipSlot, Inventory, Item}, LoadoutBuilder, }; +use i18n::{Localization, LocalizationHandle}; //ImageFrame, Tooltip, use crate::settings::Settings; //use std::time::Duration; diff --git a/voxygen/src/menu/main/mod.rs b/voxygen/src/menu/main/mod.rs index 06457e5069..910174901a 100644 --- a/voxygen/src/menu/main/mod.rs +++ b/voxygen/src/menu/main/mod.rs @@ -6,7 +6,6 @@ use super::char_selection::CharSelectionState; #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; use crate::{ - i18n::LocalizationHandle, render::{Drawer, GlobalsBindGroup}, settings::Settings, window::Event, @@ -20,6 +19,7 @@ use client::{ use client_init::{ClientInit, Error as InitError, Msg as InitMsg}; use common::comp; use common_base::span; +use i18n::LocalizationHandle; use scene::Scene; use std::sync::Arc; use tokio::runtime; diff --git a/voxygen/src/menu/main/ui/connecting.rs b/voxygen/src/menu/main/ui/connecting.rs index 42c090a1c6..49651fdb21 100644 --- a/voxygen/src/menu/main/ui/connecting.rs +++ b/voxygen/src/menu/main/ui/connecting.rs @@ -1,13 +1,12 @@ use super::{ConnectionState, Imgs, Message}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{component::neat_button, style, widget::Image, Element, IcedUi as Ui, Id}, - Graphic, - }, + +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{component::neat_button, style, widget::Image, Element, IcedUi as Ui, Id}, + Graphic, }; use common::assets::{self, AssetExt}; +use i18n::Localization; use iced::{button, Align, Column, Container, Length, Row, Space, Text}; use serde::{Deserialize, Serialize}; diff --git a/voxygen/src/menu/main/ui/disclaimer.rs b/voxygen/src/menu/main/ui/disclaimer.rs index b7225a0edf..885b29bc15 100644 --- a/voxygen/src/menu/main/ui/disclaimer.rs +++ b/voxygen/src/menu/main/ui/disclaimer.rs @@ -1,6 +1,6 @@ use super::Message; +use i18n::{Localization}; use crate::{ - i18n::Localization, ui::{ fonts::IcedFonts as Fonts, ice::{component::neat_button, style, Element}, diff --git a/voxygen/src/menu/main/ui/login.rs b/voxygen/src/menu/main/ui/login.rs index cb8303b951..9234e713d6 100644 --- a/voxygen/src/menu/main/ui/login.rs +++ b/voxygen/src/menu/main/ui/login.rs @@ -1,19 +1,17 @@ use super::{Imgs, LoginInfo, Message, FILL_FRAC_ONE, FILL_FRAC_TWO}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{ - component::neat_button, - style, - widget::{ - compound_graphic::{CompoundGraphic, Graphic}, - BackgroundContainer, Image, Padding, - }, - Element, +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{ + component::neat_button, + style, + widget::{ + compound_graphic::{CompoundGraphic, Graphic}, + BackgroundContainer, Image, Padding, }, + Element, }, }; +use i18n::{LanguageMetadata, Localization}; use iced::{ button, scrollable, text_input, Align, Button, Column, Container, Length, Row, Scrollable, Space, Text, TextInput, @@ -61,7 +59,7 @@ impl Screen { i18n: &Localization, is_selecting_language: bool, selected_language_index: Option, - language_metadatas: &[crate::i18n::LanguageMetadata], + language_metadatas: &[LanguageMetadata], button_style: style::button::Style, version: &str, ) -> Element { @@ -223,7 +221,7 @@ impl LanguageSelectBanner { fonts: &Fonts, imgs: &Imgs, i18n: &Localization, - language_metadatas: &[crate::i18n::LanguageMetadata], + language_metadatas: &[LanguageMetadata], selected_language_index: Option, button_style: style::button::Style, ) -> Element { diff --git a/voxygen/src/menu/main/ui/mod.rs b/voxygen/src/menu/main/ui/mod.rs index 2cb8772994..a013750891 100644 --- a/voxygen/src/menu/main/ui/mod.rs +++ b/voxygen/src/menu/main/ui/mod.rs @@ -5,7 +5,6 @@ mod login; mod servers; use crate::{ - i18n::{LanguageMetadata, LocalizationHandle}, render::UiDrawer, ui::{ self, @@ -16,6 +15,7 @@ use crate::{ }, window, GlobalState, }; +use i18n::{LanguageMetadata, LocalizationHandle}; use iced::{text_input, Column, Container, HorizontalAlignment, Length, Row, Space}; //ImageFrame, Tooltip, use crate::settings::Settings; @@ -193,7 +193,7 @@ impl Controls { .iter() .position(|f| f == &login_info.server); - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); let selected_language_index = language_metadatas .iter() .position(|f| f.language_identifier == settings.language.selected_language); @@ -256,7 +256,7 @@ impl Controls { self.imgs.bg }; - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); // TODO: make any large text blocks scrollable so that if the area is to // small they can still be read @@ -315,7 +315,7 @@ impl Controls { ui: &mut Ui, ) { let servers = &settings.networking.servers; - let mut language_metadatas = crate::i18n::list_localizations(); + let mut language_metadatas = i18n::list_localizations(); match message { Message::Quit => events.push(Event::Quit), @@ -509,7 +509,7 @@ impl MainMenuUi { self.ui.clear_fonts(font); self.controls.fonts = Fonts::load(i18n.fonts(), &mut self.ui).expect("Impossible to load fonts!"); - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); self.controls.selected_language_index = language_metadatas .iter() .position(|f| f.language_identifier == settings.language.selected_language); diff --git a/voxygen/src/menu/main/ui/servers.rs b/voxygen/src/menu/main/ui/servers.rs index b07804ee63..4e4f20343f 100644 --- a/voxygen/src/menu/main/ui/servers.rs +++ b/voxygen/src/menu/main/ui/servers.rs @@ -1,11 +1,9 @@ use super::{Imgs, Message, FILL_FRAC_ONE}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{component::neat_button, style, Element}, - }, +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{component::neat_button, style, Element}, }; +use i18n::Localization; use iced::{ button, scrollable, Align, Button, Column, Container, Length, Row, Scrollable, Space, Text, }; diff --git a/voxygen/src/session/mod.rs b/voxygen/src/session/mod.rs index f64aed6221..3e7bda0b77 100644 --- a/voxygen/src/session/mod.rs +++ b/voxygen/src/session/mod.rs @@ -35,6 +35,7 @@ use common_net::{ use crate::{ audio::sfx::SfxEvent, + error::Error, game_input::GameInput, hud::{DebugInfo, Event as HudEvent, Hud, HudInfo, LootMessage, PromptDialogSettings}, key_state::KeyState, @@ -43,7 +44,7 @@ use crate::{ scene::{camera, terrain::Interaction, CameraMode, DebugShapeId, Scene, SceneData}, settings::Settings, window::{AnalogGameInput, Event}, - Direction, Error, GlobalState, PlayState, PlayStateResult, + Direction, GlobalState, PlayState, PlayStateResult, }; use hashbrown::HashMap; use settings_change::Language::ChangeLanguage; diff --git a/voxygen/src/session/settings_change.rs b/voxygen/src/session/settings_change.rs index 6f8239c141..62e18e8547 100644 --- a/voxygen/src/session/settings_change.rs +++ b/voxygen/src/session/settings_change.rs @@ -6,7 +6,6 @@ use crate::{ BarNumbers, BuffPosition, ChatTab, CrosshairType, Intro, PressBehavior, ScaleChange, ShortcutNumbers, XpBar, }, - i18n::{LanguageMetadata, LocalizationHandle}, render::RenderMode, settings::{ AudioSettings, ChatSettings, ControlSettings, Fps, GamepadSettings, GameplaySettings, @@ -15,6 +14,7 @@ use crate::{ window::FullScreenSettings, GlobalState, }; +use i18n::{LanguageMetadata, LocalizationHandle}; use vek::*; #[derive(Clone)] diff --git a/voxygen/src/settings/language.rs b/voxygen/src/settings/language.rs index b8e3f8b20e..28fd1840e3 100644 --- a/voxygen/src/settings/language.rs +++ b/voxygen/src/settings/language.rs @@ -1,4 +1,3 @@ -use crate::i18n; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/voxygen/src/ui/cache.rs b/voxygen/src/ui/cache.rs index dcf5041b8e..fd807d48e6 100644 --- a/voxygen/src/ui/cache.rs +++ b/voxygen/src/ui/cache.rs @@ -1,7 +1,7 @@ use super::graphic::{Graphic, GraphicCache, Id as GraphicId}; use crate::{ + error::Error, render::{Mesh, Renderer, Texture, UiTextureBindGroup, UiVertex}, - Error, }; use conrod_core::{text::GlyphCache, widget::Id}; use hashbrown::HashMap; diff --git a/voxygen/src/ui/fonts.rs b/voxygen/src/ui/fonts.rs index 1bd9ac45a1..fe3716542d 100644 --- a/voxygen/src/ui/fonts.rs +++ b/voxygen/src/ui/fonts.rs @@ -1,4 +1,4 @@ -use crate::{i18n, ui::ice::RawFont}; +use crate::ui::ice::RawFont; use common::assets::{self, AssetExt}; pub struct Font { diff --git a/voxygen/src/ui/ice/cache.rs b/voxygen/src/ui/ice/cache.rs index aeb629bc88..e7631e10a4 100644 --- a/voxygen/src/ui/ice/cache.rs +++ b/voxygen/src/ui/ice/cache.rs @@ -1,7 +1,7 @@ use super::graphic::{Graphic, GraphicCache, Id as GraphicId}; use crate::{ + error::Error, render::{Renderer, Texture, UiTextureBindGroup}, - Error, }; use common::assets::{self, AssetExt}; use glyph_brush::GlyphBrushBuilder; diff --git a/voxygen/src/ui/ice/mod.rs b/voxygen/src/ui/ice/mod.rs index 91393ad0cf..d28ef981d9 100644 --- a/voxygen/src/ui/ice/mod.rs +++ b/voxygen/src/ui/ice/mod.rs @@ -15,9 +15,9 @@ use super::{ scale::{Scale, ScaleMode}, }; use crate::{ + error::Error, render::{Renderer, UiDrawer}, window::Window, - Error, }; use common::slowjob::SlowJobPool; use common_base::span; diff --git a/voxygen/src/ui/ice/renderer/mod.rs b/voxygen/src/ui/ice/renderer/mod.rs index 51f4fbf494..2c6ff6014c 100644 --- a/voxygen/src/ui/ice/renderer/mod.rs +++ b/voxygen/src/ui/ice/renderer/mod.rs @@ -14,11 +14,11 @@ use super::{ Font, FontId, RawFont, Rotation, }; use crate::{ + error::Error, render::{ create_ui_quad, create_ui_quad_vert_gradient, DynamicModel, Mesh, Renderer, UiBoundLocals, UiDrawer, UiLocals, UiMode, UiVertex, }, - Error, }; use common::{slowjob::SlowJobPool, util::srgba_to_linear}; use common_base::span; diff --git a/voxygen/src/ui/mod.rs b/voxygen/src/ui/mod.rs index 574dd6ed8c..65b57fb139 100644 --- a/voxygen/src/ui/mod.rs +++ b/voxygen/src/ui/mod.rs @@ -28,12 +28,12 @@ pub use widgets::{ }; use crate::{ + error::Error, render::{ create_ui_quad, create_ui_tri, DynamicModel, Mesh, RenderError, Renderer, UiBoundLocals, UiDrawer, UiLocals, UiMode, UiVertex, }, window::Window, - Error, }; #[rustfmt::skip] use ::image::GenericImageView; diff --git a/voxygen/src/ui/widgets/item_tooltip.rs b/voxygen/src/ui/widgets/item_tooltip.rs index 4dcd6f7cf0..86c3ef1ad4 100644 --- a/voxygen/src/ui/widgets/item_tooltip.rs +++ b/voxygen/src/ui/widgets/item_tooltip.rs @@ -1,12 +1,9 @@ use super::image_frame::ImageFrame; -use crate::{ - hud::{ - get_quality_col, - img_ids::Imgs, - item_imgs::{animate_by_pulse, ItemImgs, ItemKey}, - util, - }, - i18n::Localization, +use crate::hud::{ + get_quality_col, + img_ids::Imgs, + item_imgs::{animate_by_pulse, ItemImgs, ItemKey}, + util, }; use client::Client; use common::{ @@ -22,6 +19,7 @@ use conrod_core::{ widget, widget_ids, Color, Colorable, FontSize, Positionable, Scalar, Sizeable, Ui, UiCell, Widget, WidgetCommon, WidgetStyle, }; +use i18n::Localization; use lazy_static::lazy_static; use std::time::{Duration, Instant}; diff --git a/voxygen/src/window.rs b/voxygen/src/window.rs index 3663e3443a..4f2c3da9d4 100644 --- a/voxygen/src/window.rs +++ b/voxygen/src/window.rs @@ -1,9 +1,10 @@ use crate::{ controller::*, + error::Error, game_input::GameInput, render::Renderer, settings::{ControlSettings, Settings}, - ui, Error, + ui, }; use common_base::span; use crossbeam_channel as channel; From c234f1d9266dc0c50b685e763fceffa87014d871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Thu, 29 Jul 2021 21:26:37 +0200 Subject: [PATCH 7/7] mark missing files as untranslated --- voxygen/i18n/src/analysis.rs | 25 +++++++++++++++++++------ voxygen/i18n/src/gitfragments.rs | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index e15d298839..28b055c908 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -69,6 +69,13 @@ fn compare_lang_with_reference( } }; + const MISSING: LocalizationEntryState = LocalizationEntryState { + key_line: None, + chuck_line_range: None, + commit_id: None, + state: Some(LocalizationState::NotFound), + }; + // match files for (ref_path, ref_fragment) in i18n_references.fragments.iter() { let cur_fragment = match current_i18n.fragments.get_mut(ref_path) { @@ -78,6 +85,17 @@ fn compare_lang_with_reference( "language {} is missing file: {:?}", current_i18n.manifest.metadata.language_identifier, ref_path ); + // add all keys as missing + let mut string_map = HashMap::new(); + for (ref_key, _) in ref_fragment.string_map.iter() { + string_map.insert(ref_key.to_owned(), MISSING.clone()); + } + current_i18n + .fragments + .insert(ref_path.to_owned(), RawFragment { + string_map, + vector_map: HashMap::new(), + }); continue; }, }; @@ -118,12 +136,7 @@ fn compare_lang_with_reference( None => { cur_fragment .string_map - .insert(ref_key.to_owned(), LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: Some(LocalizationState::NotFound), - }); + .insert(ref_key.to_owned(), MISSING.clone()); }, } } diff --git a/voxygen/i18n/src/gitfragments.rs b/voxygen/i18n/src/gitfragments.rs index e517360d42..ec77568837 100644 --- a/voxygen/i18n/src/gitfragments.rs +++ b/voxygen/i18n/src/gitfragments.rs @@ -19,7 +19,7 @@ pub(crate) const ALL_LOCALIZATION_STATES: [Option; 5] = [ None, ]; -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Debug)] pub(crate) struct LocalizationEntryState { pub(crate) key_line: Option, pub(crate) chuck_line_range: Option<(usize, usize)>,