diff --git a/.gitlab/scripts/unittest.sh b/.gitlab/scripts/unittest.sh index a4193d3914..4bb46b4995 100755 --- a/.gitlab/scripts/unittest.sh +++ b/.gitlab/scripts/unittest.sh @@ -1,7 +1,7 @@ #!/bin/bash export VELOREN_ASSETS="$(pwd)/assets" rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly -time cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored && +time cargo test --package veloren-voxygen-i18n --lib test_all_localizations -- --nocapture --ignored && time cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib && ( rm -r target/debug/incremental* || echo "all good" ) && # TMP FIX FOR 2021-03-22-nightly time cargo test \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0274d8ca78..3132f6e3fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6017,20 +6017,6 @@ dependencies = [ "veloren-common-net", ] -[[package]] -name = "veloren-i18n" -version = "0.10.0" -dependencies = [ - "clap", - "deunicode", - "git2", - "hashbrown 0.11.2", - "ron", - "serde", - "tracing", - "veloren-common-assets", -] - [[package]] name = "veloren-network" version = "0.3.0" @@ -6236,10 +6222,10 @@ dependencies = [ "veloren-common-net", "veloren-common-state", "veloren-common-systems", - "veloren-i18n", "veloren-server", "veloren-voxygen-anim", "veloren-voxygen-egui", + "veloren-voxygen-i18n", "veloren-world", "wgpu", "wgpu-profiler", @@ -6295,6 +6281,20 @@ dependencies = [ "veloren-voxygen-egui", ] +[[package]] +name = "veloren-voxygen-i18n" +version = "0.10.0" +dependencies = [ + "clap", + "deunicode", + "git2", + "hashbrown 0.11.2", + "ron", + "serde", + "tracing", + "veloren-common-assets", +] + [[package]] name = "veloren-world" version = "0.10.0" diff --git a/voxygen/Cargo.toml b/voxygen/Cargo.toml index 924be04100..c7504d5890 100644 --- a/voxygen/Cargo.toml +++ b/voxygen/Cargo.toml @@ -46,7 +46,7 @@ common-systems = {package = "veloren-common-systems", path = "../common/systems" common-state = {package = "veloren-common-state", path = "../common/state"} anim = {package = "veloren-voxygen-anim", path = "anim"} -i18n = {package = "veloren-i18n", path = "i18n"} +i18n = {package = "veloren-voxygen-i18n", path = "i18n"} voxygen-egui = {package = "veloren-voxygen-egui", path = "egui", optional = true } # Graphics diff --git a/voxygen/i18n/Cargo.toml b/voxygen/i18n/Cargo.toml index 388586679b..5ebbbb05d1 100644 --- a/voxygen/i18n/Cargo.toml +++ b/voxygen/i18n/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["juliancoffee ", "Rémy Phelipot"] edition = "2018" -name = "veloren-i18n" +name = "veloren-voxygen-i18n" description = "Crate for internalization and diagnostic of existing localizations." version = "0.10.0" diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index dcf8ec3f11..28b055c908 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -1,648 +1,240 @@ -use ron::de::from_bytes; -use std::path::{Path, PathBuf}; - -use crate::data::{ - i18n_directories, LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG, +use crate::{ + gitfragments::{ + read_file_from_path, transform_fragment, LocalizationEntryState, LocalizationState, + }, + path::{BasePath, LangPath}, + raw::{self, RawFragment, RawLanguage}, + stats::{ + print_csv_file, print_overall_stats, print_translation_stats, LocalizationAnalysis, + LocalizationStats, + }, + REFERENCE_LANG, }; -use hashbrown::{HashMap, HashSet}; +use hashbrown::{hash_map::Entry, HashMap}; +use ron::de::from_bytes; -#[derive(Copy, Clone, Eq, Hash, Debug, PartialEq)] -enum LocalizationState { - UpToDate, - NotFound, - Outdated, - Unknown, - Unused, -} - -#[derive(Debug, PartialEq)] -struct LocalizationStats { - uptodate_entries: usize, - outdated_entries: usize, - unused_entries: usize, - notfound_entries: usize, - errors: usize, - real_entry_count: usize, -} - -#[derive(Default)] -struct LocalizationAnalysis { - notfound: Vec<(String, Option)>, - unused: Vec<(String, Option)>, - outdated: Vec<(String, Option)>, - unknown: Vec<(String, Option)>, -} - -impl LocalizationAnalysis { - fn get_mut( - &mut self, - state: LocalizationState, - ) -> Option<&mut Vec<(String, Option)>> { - match state { - LocalizationState::NotFound => Some(&mut self.notfound), - LocalizationState::Unused => Some(&mut self.unused), - LocalizationState::Outdated => Some(&mut self.outdated), - LocalizationState::Unknown => Some(&mut self.unknown), - _ => None, - } - } - - fn show( - &mut self, - state: LocalizationState, - be_verbose: bool, - ref_i18n_map: &HashMap, - ) { - let entries = self - .get_mut(state) - .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); - if entries.is_empty() { - return; - } - println!("\n\t[{:?}]", state); - entries.sort(); - for (key, commit_id) in entries { - if be_verbose { - let our_commit = commit_id - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()); - let ref_commit = ref_i18n_map - .get(key) - .and_then(|s| s.commit_id) - .map(|s| format!("{}", s)) - .unwrap_or_else(|| "None".to_owned()); - println!("{:60}| {:40} | {:40}", key, our_commit, ref_commit,); - } else { - println!("{}", key); - } - } - } -} - -#[derive(Copy, Clone, Debug)] -struct LocalizationEntryState { - key_line: Option, - chuck_line_range: Option<(usize, usize)>, - commit_id: Option, - state: LocalizationState, -} - -impl LocalizationEntryState { - fn new() -> LocalizationEntryState { - LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::Unknown, - } - } -} - -/// Returns the Git blob associated with the given reference and path -fn read_file_from_path<'a>( - repo: &'a git2::Repository, - reference: &git2::Reference, - path: &std::path::Path, -) -> git2::Blob<'a> { - let tree = reference - .peel_to_tree() - .expect("Impossible to peel HEAD to a tree object"); - tree.get_path(path) - .unwrap_or_else(|_| { - panic!( - "Impossible to find the file {:?} in reference {:?}", - path, - reference.name() - ) - }) - .to_object(repo) - .unwrap() - .peel_to_blob() - .expect("Impossible to fetch the Git object") -} - -fn correspond(line: &str, key: &str) -> bool { - let pat = { - // Get left part of split - let mut begin = line - .split(':') - .next() - .expect("split always produces value") - .trim() - .chars(); - // Remove quotes - begin.next(); - begin.next_back(); - begin.as_str() - }; - - pat == key -} - -fn generate_key_version<'a>( - repo: &'a git2::Repository, - localization: &LocalizationFragment, - path: &std::path::Path, - file_blob: &git2::Blob, -) -> HashMap { - let mut keys: HashMap = localization - .string_map - .keys() - .map(|k| (k.to_owned(), LocalizationEntryState::new())) - .collect(); - // Find key start lines - let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); - let mut to_process: HashSet<&String> = localization.string_map.keys().collect(); - for (line_nb, line) in file_content.lines().enumerate() { - let mut found_key = None; - - for key in to_process.iter() { - if correspond(line, key) { - found_key = Some(key.to_owned()); - } - } - - if let Some(key) = found_key { - keys.get_mut(key).unwrap().key_line = Some(line_nb); - to_process.remove(key); - }; - } - - let mut error_check_set: Vec = vec![]; - // Find commit for each keys - repo.blame_file(path, None) - .expect("Impossible to generate the Git blame") - .iter() - .for_each(|e: git2::BlameHunk| { - for (key, state) in keys.iter_mut() { - let line = match state.key_line { - Some(l) => l, - None => { - if !error_check_set.contains(key) { - eprintln!( - "Key {} does not have a git line in it's state! Skipping key.", - key - ); - error_check_set.push(key.clone()); - } - continue; - }, - }; - - if line + 1 >= e.final_start_line() - && line + 1 < e.final_start_line() + e.lines_in_hunk() - { - state.chuck_line_range = Some(( - e.final_start_line(), - e.final_start_line() + e.lines_in_hunk(), - )); - state.commit_id = match state.commit_id { - Some(existing_commit) => { - match repo.graph_descendant_of(e.final_commit_id(), existing_commit) { - Ok(true) => Some(e.final_commit_id()), - Ok(false) => Some(existing_commit), - Err(err) => panic!("{}", err), - } - }, - None => Some(e.final_commit_id()), - }; - } - } - }); - - keys -} - -fn complete_key_versions<'a>( +/// Fill the entry State base information (except `state`) for a complete +/// language +fn gather_entry_state<'a>( repo: &'a git2::Repository, head_ref: &git2::Reference, - i18n_key_versions: &mut HashMap, - root_dir: &Path, - lang_dir: &Path, -) { - //TODO: review unwraps in this file + path: &LangPath, +) -> RawLanguage { + println!("-> {:?}", path.language_identifier()); + // load standard manifest + let manifest = raw::load_manifest(path).expect("failed to load language manifest"); + // transform language into LocalizationEntryState + let mut fragments = HashMap::new(); // For each file in directory - for i18n_file in root_dir.join(&lang_dir).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - if file_type.is_file() { - println!("-> {:?}", i18n_file.file_name()); + let files = path + .fragments() + .expect("failed to get all files in language"); + for sub_path in files { + let fullpath = path.sub_path(&sub_path); + let gitpath = fullpath.strip_prefix(path.base().root_path()).unwrap(); + println!(" -> {:?}", &sub_path); + let i18n_blob = read_file_from_path(repo, head_ref, gitpath); + let fragment: RawFragment = from_bytes(i18n_blob.content()) + .unwrap_or_else(|e| panic!("Could not parse {:?} RON file, error: {}", sub_path, e)); + let frag = transform_fragment(repo, (gitpath, fragment), &i18n_blob); + fragments.insert(sub_path, frag); + } - let full_path = i18n_file.path(); - let path = full_path.strip_prefix(root_dir).unwrap(); - let i18n_blob = read_file_from_path(repo, head_ref, path); - let i18n: LocalizationFragment = - from_bytes(i18n_blob.content()).unwrap_or_else(|e| { - panic!( - "Could not parse {} RON file, skipping: {}", - i18n_file.path().to_string_lossy(), - e - ) - }); - i18n_key_versions.extend(generate_key_version(repo, &i18n, path, &i18n_blob)); - } else if file_type.is_dir() { - // If it's a directory, recursively check it - complete_key_versions( - repo, - head_ref, - i18n_key_versions, - root_dir, - &i18n_file.path(), + RawLanguage:: { + manifest, + fragments, + } +} + +/// fills in the `state` +fn compare_lang_with_reference( + current_i18n: &mut RawLanguage, + i18n_references: &RawLanguage, + repo: &git2::Repository, +) { + // git graph decendent of is slow, so we cache it + let mut graph_decendent_of_cache = HashMap::new(); + + let mut cached_graph_descendant_of = |commit, ancestor| -> bool { + let key = (commit, ancestor); + match graph_decendent_of_cache.entry(key) { + Entry::Occupied(entry) => { + return *entry.get(); + }, + Entry::Vacant(entry) => { + let value = repo.graph_descendant_of(commit, ancestor).unwrap_or(false); + *entry.insert(value) + }, + } + }; + + const MISSING: LocalizationEntryState = LocalizationEntryState { + key_line: None, + chuck_line_range: None, + commit_id: None, + state: Some(LocalizationState::NotFound), + }; + + // match files + for (ref_path, ref_fragment) in i18n_references.fragments.iter() { + let cur_fragment = match current_i18n.fragments.get_mut(ref_path) { + Some(c) => c, + None => { + eprintln!( + "language {} is missing file: {:?}", + current_i18n.manifest.metadata.language_identifier, ref_path ); + // add all keys as missing + let mut string_map = HashMap::new(); + for (ref_key, _) in ref_fragment.string_map.iter() { + string_map.insert(ref_key.to_owned(), MISSING.clone()); + } + current_i18n + .fragments + .insert(ref_path.to_owned(), RawFragment { + string_map, + vector_map: HashMap::new(), + }); + continue; + }, + }; + + for (ref_key, ref_state) in ref_fragment.string_map.iter() { + match cur_fragment.string_map.get_mut(ref_key) { + Some(state) => { + let commit_id = match state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in i18n file {} is missing! Skipping key.", + ref_key, + ref_path.to_string_lossy() + ); + continue; + }, + }; + let ref_commit_id = match ref_state.commit_id { + Some(c) => c, + None => { + eprintln!( + "Commit ID of key {} in reference i18n file is missing! Skipping \ + key.", + ref_key + ); + continue; + }, + }; + if commit_id != ref_commit_id + && !cached_graph_descendant_of(commit_id, ref_commit_id) + { + state.state = Some(LocalizationState::Outdated); + } else { + state.state = Some(LocalizationState::UpToDate); + } + }, + None => { + cur_fragment + .string_map + .insert(ref_key.to_owned(), MISSING.clone()); + }, } } - } -} -fn gather_state( - loc: &RawLocalization, - i18n_blob: &git2::Blob, - ref_manifest: &Path, - root_dir: &Path, - lang_dir: &Path, - repo: &git2::Repository, - head_ref: &git2::Reference, -) -> HashMap { - // Generate map - let mut i18n_map = generate_key_version( - repo, - &LocalizationFragment::from(loc.clone()), - ref_manifest, - i18n_blob, - ); - - // Gathering info about keys from language - complete_key_versions(repo, head_ref, &mut i18n_map, root_dir, lang_dir); - - i18n_map -} - -// Helper function to test localization directory -// `lang_dir` - path to localization directory. Relative from root of the -// repo. -// `root_dir` - absolute path to repo -// `ref_manifest` - path to reference manifest -// `i18n_references` - keys from reference language -// `repo` - git object for main repo -// `head_ref` - HEAD -fn test_localization_directory( - lang_dir: &Path, - root_dir: &Path, - ref_manifest: &Path, - i18n_references: &HashMap, - be_verbose: bool, - repo: &git2::Repository, - head_ref: &git2::Reference, -) -> Option { - let relfile = lang_dir.join(&(LANG_MANIFEST_FILE.to_string() + ".ron")); - if relfile == ref_manifest { - return None; - } - println!("\n-----------------------------------"); - println!("{:?}", relfile); - println!("-----------------------------------"); - - // Find the localization entry state - let current_blob = read_file_from_path(repo, head_ref, &relfile); - let current_loc: RawLocalization = from_bytes(current_blob.content()).unwrap_or_else(|e| { - panic!( - "Could not parse {} RON file, skipping: {}", - relfile.to_string_lossy(), - e - ) - }); - - // Gather state of current localization - let mut current_i18n = gather_state( - ¤t_loc, - ¤t_blob, - ref_manifest, - root_dir, - lang_dir, - repo, - head_ref, - ); - - // Comparing with reference localization - fill_info(&mut current_i18n, i18n_references, repo, &relfile); - - let mut state_map = LocalizationAnalysis::default(); - let result = gather_results(current_i18n, &mut state_map); - print_translation_stats( - i18n_references, - &result, - &mut state_map, - be_verbose, - relfile, - ref_manifest, - ); - Some(result) -} - -fn fill_info( - current_i18n: &mut HashMap, - i18n_references: &HashMap, - repo: &git2::Repository, - relfile: &Path, -) { - for (ref_key, ref_state) in i18n_references.iter() { - match current_i18n.get_mut(ref_key) { - Some(state) => { - let commit_id = match state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in i18n file {} is missing! Skipping key.", - ref_key, - relfile.to_string_lossy() - ); - continue; - }, - }; - let ref_commit_id = match ref_state.commit_id { - Some(c) => c, - None => { - eprintln!( - "Commit ID of key {} in reference i18n file is missing! Skipping key.", - ref_key - ); - continue; - }, - }; - if commit_id != ref_commit_id - && !repo - .graph_descendant_of(commit_id, ref_commit_id) - .unwrap_or(false) - { - state.state = LocalizationState::Outdated; - } else { - state.state = LocalizationState::UpToDate; - } - }, - None => { - current_i18n.insert(ref_key.to_owned(), LocalizationEntryState { - key_line: None, - chuck_line_range: None, - commit_id: None, - state: LocalizationState::NotFound, - }); - }, + for (_, state) in cur_fragment + .string_map + .iter_mut() + .filter(|&(k, _)| ref_fragment.string_map.get(k).is_none()) + { + state.state = Some(LocalizationState::Unused); } } - - let ref_keys: HashSet<&String> = i18n_references.keys().collect(); - for (_, state) in current_i18n - .iter_mut() - .filter(|&(k, _)| !ref_keys.contains(k)) - { - state.state = LocalizationState::Unused; - } } fn gather_results( - current_i18n: HashMap, - state_map: &mut LocalizationAnalysis, -) -> LocalizationStats { - let mut uptodate_entries = 0; - let mut outdated_entries = 0; - let mut unused_entries = 0; - let mut notfound_entries = 0; - let mut unknown_entries = 0; + current_i18n: &RawLanguage, +) -> (LocalizationAnalysis, LocalizationStats) { + let mut state_map = + LocalizationAnalysis::new(¤t_i18n.manifest.metadata.language_identifier); + let mut stats = LocalizationStats::default(); - let keys: Vec<&String> = current_i18n.keys().collect(); - for key in keys { - let entry = current_i18n.get(key).unwrap(); - match entry.state { - LocalizationState::Outdated => outdated_entries += 1, - LocalizationState::NotFound => notfound_entries += 1, - LocalizationState::Unknown => unknown_entries += 1, - LocalizationState::Unused => unused_entries += 1, - LocalizationState::UpToDate => uptodate_entries += 1, - }; - if entry.state != LocalizationState::UpToDate { - let state_keys = state_map - .get_mut(entry.state) - .expect("vectors must be added"); - state_keys.push((key.to_owned(), entry.commit_id)); + for (file, fragments) in ¤t_i18n.fragments { + for (key, entry) in &fragments.string_map { + match entry.state { + Some(LocalizationState::Outdated) => stats.outdated_entries += 1, + Some(LocalizationState::NotFound) => stats.notfound_entries += 1, + None => stats.errors += 1, + Some(LocalizationState::Unused) => stats.unused_entries += 1, + Some(LocalizationState::UpToDate) => stats.uptodate_entries += 1, + }; + if entry.state != Some(LocalizationState::UpToDate) { + let state_keys = state_map.data.get_mut(&entry.state).expect("prefiled"); + state_keys.push((file.clone(), key.to_owned(), entry.commit_id)); + } } } - // Calculate key count that actually matter for the status of the translation - // Unused entries don't break the game - let current_i18n_entry_count = current_i18n.len(); - let real_entry_count = current_i18n_entry_count - unused_entries; - - LocalizationStats { - uptodate_entries, - unused_entries, - outdated_entries, - notfound_entries, - errors: unknown_entries, - real_entry_count, - } -} - -fn print_translation_stats( - ref_i18n_map: &HashMap, - stats: &LocalizationStats, - state_map: &mut LocalizationAnalysis, - be_verbose: bool, - relfile: PathBuf, - ref_manifest: &Path, -) { - let uptodate_percent = - (stats.uptodate_entries as f32 / stats.real_entry_count as f32) * 100_f32; - let outdated_percent = - (stats.outdated_entries as f32 / stats.real_entry_count as f32) * 100_f32; - let untranslated_percent = - ((stats.errors + stats.errors) as f32 / stats.real_entry_count as f32) * 100_f32; - - // Display - if be_verbose { - println!( - "\n{:60}| {:40} | {:40}", - "Key name", - relfile.to_str().unwrap(), - ref_manifest.to_str().unwrap(), - ); - } else { - println!("\nKey name"); + for (_, entries) in state_map.data.iter_mut() { + entries.sort(); } - state_map.show(LocalizationState::NotFound, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Unused, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Outdated, be_verbose, ref_i18n_map); - state_map.show(LocalizationState::Unknown, be_verbose, ref_i18n_map); - - println!( - "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", - stats.uptodate_entries, - stats.outdated_entries, - stats.unused_entries, - stats.notfound_entries, - stats.errors, - ); - - println!( - "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", - uptodate_percent, outdated_percent, untranslated_percent, - ); + (state_map, stats) } /// Test one language -/// `code` - name of the directory in assets (de_DE for example) -/// `root_dir` - absolute path to main repo -/// `assets_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -pub fn test_specific_localization( - code: &str, - root_dir: &Path, - assets_path: &Path, +/// - `code`: name of the directory in assets (de_DE for example) +/// - `path`: path to repo +/// - `be_verbose`: print extra info +/// - `csv_enabled`: generate csv files in target folder +pub fn test_specific_localizations( + path: &BasePath, + language_identifiers: &[&str], be_verbose: bool, + csv_enabled: bool, ) { - // Relative paths from root of repo to assets - let ref_lang_dir = assets_path.join(REFERENCE_LANG); - let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - + //complete analysis + let mut analysis = HashMap::new(); // Initialize Git objects - let repo = git2::Repository::discover(&root_dir) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); + let repo = git2::Repository::discover(path.root_path()) + .unwrap_or_else(|_| panic!("Failed to open the Git repository {:?}", path.root_path())); let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - // Read HEAD for the reference language manifest - let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest); - let loc: RawLocalization = from_bytes(ref_manifest_blob.content()) - .expect("Expect to parse reference i18n RON file, can't proceed without it"); + // Read Reference Language + let ref_language = gather_entry_state(&repo, &head_ref, &path.i18n_path(REFERENCE_LANG)); + for &language_identifier in language_identifiers { + let mut cur_language = + gather_entry_state(&repo, &head_ref, &path.i18n_path(language_identifier)); + compare_lang_with_reference(&mut cur_language, &ref_language, &repo); + let (state_map, stats) = gather_results(&cur_language); + analysis.insert(language_identifier.to_owned(), (state_map, stats)); + } - // Gathering info about keys from reference language - let reference_i18n = gather_state( - &loc, - &ref_manifest_blob, - &ref_manifest, - root_dir, - &ref_lang_dir, - &repo, - &head_ref, - ); - - // Testing how specific language is localized - let dir = assets_path.join(code); - test_localization_directory( - &dir, - root_dir, - &ref_manifest, - &reference_i18n, - be_verbose, - &repo, - &head_ref, - ); + //printing + for (language_identifier, (state_map, stats)) in &analysis { + if csv_enabled { + print_csv_file(state_map); + } else { + print_translation_stats( + language_identifier, + &ref_language, + stats, + state_map, + be_verbose, + ); + } + } + if analysis.len() > 1 { + print_overall_stats(analysis); + } } /// Test all localizations -/// `root_dir` - absolute path to main repo -/// `assets_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool) { - let ref_lang_dir = assets_path.join(REFERENCE_LANG); - let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron"); - - if !root_dir.join(&ref_lang_dir).is_dir() { - panic!("Reference language folder not found {:?}", &ref_lang_dir) - } - if !root_dir.join(&ref_manifest).is_file() { - panic!("Reference language file not found {:?}", &ref_manifest) - } - - // Initialize Git objects - let repo = git2::Repository::discover(&root_dir) - .unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir)); - let head_ref = repo.head().expect("Impossible to get the HEAD reference"); - - // Read HEAD for the reference language file - let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest); - let loc: RawLocalization = from_bytes(ref_manifest_blob.content()) - .expect("Expect to parse reference i18n RON file, can't proceed without it"); - - // Gathering info about keys from reference language - let reference_i18n = gather_state( - &loc, - &ref_manifest_blob, - &ref_manifest, - root_dir, - &ref_lang_dir, - &repo, - &head_ref, - ); - +pub fn test_all_localizations(path: &BasePath, be_verbose: bool, csv_enabled: bool) { // Compare to other reference files - let i18n_directories = i18n_directories(&root_dir.join(assets_path)); - let mut i18n_entry_counts: HashMap = HashMap::new(); - for dir in &i18n_directories { - let rel_dir = dir.strip_prefix(root_dir).unwrap(); - let result = test_localization_directory( - rel_dir, - root_dir, - &ref_manifest, - &reference_i18n, - be_verbose, - &repo, - &head_ref, - ); - if let Some(values) = result { - i18n_entry_counts.insert(dir.clone(), values); - } - } - - print_overall_stats(i18n_entry_counts); -} - -fn print_overall_stats(i18n_entry_counts: HashMap) { - let mut overall_uptodate_entry_count = 0; - let mut overall_outdated_entry_count = 0; - let mut overall_untranslated_entry_count = 0; - let mut overall_real_entry_count = 0; - - println!("-----------------------------------------------------------------------------"); - println!("Overall Translation Status"); - println!("-----------------------------------------------------------------------------"); - println!( - "{:12}| {:8} | {:8} | {:8} | {:8} | {:8}", - "", "up-to-date", "outdated", "untranslated", "unused", "errors", - ); - - let mut i18n_stats: Vec<(&PathBuf, &LocalizationStats)> = i18n_entry_counts.iter().collect(); - i18n_stats.sort_by_key(|(_, result)| result.notfound_entries); - - for (path, test_result) in i18n_stats { - let LocalizationStats { - uptodate_entries: uptodate, - outdated_entries: outdated, - unused_entries: unused, - notfound_entries: untranslated, - errors, - real_entry_count: real, - } = test_result; - overall_uptodate_entry_count += uptodate; - overall_outdated_entry_count += outdated; - overall_untranslated_entry_count += untranslated; - overall_real_entry_count += real; - - println!( - "{:12}|{:8} |{:6} |{:8} |{:6} |{:8}", - path.file_name().unwrap().to_string_lossy(), - uptodate, - outdated, - untranslated, - unused, - errors, - ); - } - - println!( - "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", - (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, - ); - println!("-----------------------------------------------------------------------------\n"); + let languages = path.i18n_directories(); + let language_identifiers = languages + .iter() + .map(|s| s.language_identifier()) + .collect::>(); + test_specific_localizations(path, &language_identifiers, be_verbose, csv_enabled); } diff --git a/voxygen/i18n/src/bin/i18n-check.rs b/voxygen/i18n/src/bin/i18n-check.rs index cc45d256bb..6a868a950f 100644 --- a/voxygen/i18n/src/bin/i18n-check.rs +++ b/voxygen/i18n/src/bin/i18n-check.rs @@ -1,6 +1,5 @@ use clap::{App, Arg}; -use std::path::Path; -use veloren_i18n::{analysis, verification}; +use veloren_voxygen_i18n::{analysis, verification, BasePath}; fn main() { let matches = App::new("i18n-check") @@ -28,24 +27,26 @@ fn main() { .long("verbose") .help("print additional information"), ) + .arg( + Arg::with_name("csv") + .long("csv") + .help("generate csv files per language in target folder"), + ) .get_matches(); // Generate paths - let root = common_assets::find_root().expect("Failed to find root of repository"); - let asset_path = Path::new("assets/voxygen/i18n/"); + let root_path = common_assets::find_root().expect("Failed to find root of repository"); + let path = BasePath::new(&root_path); + let be_verbose = matches.is_present("verbose"); + let csv_enabled = matches.is_present("csv"); if let Some(code) = matches.value_of("CODE") { - analysis::test_specific_localization( - code, - &root, - &asset_path, - matches.is_present("verbose"), - ); + analysis::test_specific_localizations(&path, &[code], be_verbose, csv_enabled); } if matches.is_present("test") { - analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose")); + analysis::test_all_localizations(&path, be_verbose, csv_enabled); } if matches.is_present("verify") { - verification::verify_all_localizations(&root, &asset_path); + verification::verify_all_localizations(&path); } } diff --git a/voxygen/i18n/src/data.rs b/voxygen/i18n/src/data.rs deleted file mode 100644 index 2240134fe2..0000000000 --- a/voxygen/i18n/src/data.rs +++ /dev/null @@ -1,437 +0,0 @@ -use crate::assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; -use deunicode::deunicode; -use hashbrown::{HashMap, HashSet}; -use serde::{Deserialize, Serialize}; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; -use tracing::warn; - -/// The reference language, aka the more up-to-date localization data. -/// Also the default language at first startup. -pub const REFERENCE_LANG: &str = "en"; - -pub const LANG_MANIFEST_FILE: &str = "_manifest"; - -/// How a language can be described -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LanguageMetadata { - /// A human friendly language name (e.g. "English (US)") - pub language_name: String, - - /// A short text identifier for this language (e.g. "en_US") - /// - /// On the opposite of `language_name` that can change freely, - /// `language_identifier` value shall be stable in time as it - /// is used by setting components to store the language - /// selected by the user. - pub language_identifier: String, -} - -/// Store font metadata -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Font { - /// Key to retrieve the font in the asset system - pub asset_key: String, - - /// Scale ratio to resize the UI text dynamicly - scale_ratio: f32, -} - -impl Font { - /// Scale input size to final UI size - pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } -} - -/// Store font metadata -pub type Fonts = HashMap; - -/// Raw localization data, expect the strings to not be loaded here -/// However, metadata informations are correct -/// See `Language` for more info on each attributes -#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] -pub(crate) struct RawLocalization { - pub(crate) convert_utf8_to_ascii: bool, - pub(crate) fonts: Fonts, - pub(crate) metadata: LanguageMetadata, - pub(crate) string_map: HashMap, - pub(crate) vector_map: HashMap>, -} - -/// Store internationalization data -#[derive(Debug, PartialEq, Serialize, Deserialize)] -struct Language { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, - - /// Whether to convert the input text encoded in UTF-8 - /// into a ASCII version by using the `deunicode` crate. - pub(crate) convert_utf8_to_ascii: bool, - - /// Font configuration is stored here - pub(crate) fonts: Fonts, - - pub(crate) metadata: LanguageMetadata, -} - -/// Store internationalization maps -/// These structs are meant to be merged into a Language -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub(crate) struct LocalizationFragment { - /// A map storing the localized texts - /// - /// Localized content can be accessed using a String key. - pub(crate) string_map: HashMap, - - /// A map for storing variations of localized texts, for example multiple - /// ways of saying "Help, I'm under attack". Used primarily for npc - /// dialogue. - pub(crate) vector_map: HashMap>, -} - -impl Language { - /// Get a localized text from the given key - pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { - self.string_map.get(key).map(String::as_str) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { - self.vector_map.get(key).and_then(|v| { - if v.is_empty() { - None - } else { - Some(v[index as usize % v.len()].as_str()) - } - }) - } -} - -impl Default for Language { - fn default() -> Self { - Self { - string_map: HashMap::default(), - vector_map: HashMap::default(), - ..Default::default() - } - } -} - -impl From for Language { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - convert_utf8_to_ascii: raw.convert_utf8_to_ascii, - fonts: raw.fonts, - metadata: raw.metadata, - } - } -} -impl From for LocalizationFragment { - fn from(raw: RawLocalization) -> Self { - Self { - string_map: raw.string_map, - vector_map: raw.vector_map, - } - } -} - -impl assets::Asset for RawLocalization { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} -impl assets::Asset for LocalizationFragment { - type Loader = assets::RonLoader; - - const EXTENSION: &'static str = "ron"; -} - -impl assets::Compound for Language { - fn load( - cache: &assets::AssetCache, - asset_key: &str, - ) -> Result { - let raw = cache - .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? - .cloned(); - let mut localization = Language::from(raw); - - // Walk through files in the folder, collecting localization fragment to merge - // inside the asked_localization - for localization_asset in cache - .load_dir::(asset_key, true)? - .iter() - { - localization - .string_map - .extend(localization_asset.read().string_map.clone()); - localization - .vector_map - .extend(localization_asset.read().vector_map.clone()); - } - - // Update the text if UTF-8 to ASCII conversion is enabled - if localization.convert_utf8_to_ascii { - for value in localization.string_map.values_mut() { - *value = deunicode(value); - } - - for value in localization.vector_map.values_mut() { - *value = value.iter().map(|s| deunicode(s)).collect(); - } - } - localization.metadata.language_name = deunicode(&localization.metadata.language_name); - - Ok(localization) - } -} - -/// the central data structure to handle localization in veloren -// inherit Copy+Clone from AssetHandle -#[derive(Debug, PartialEq, Copy, Clone)] -pub struct LocalizationHandle { - active: AssetHandle, - fallback: Option>, - pub use_english_fallback: bool, -} - -// RAII guard returned from Localization::read(), resembles AssetGuard -pub struct LocalizationGuard { - active: AssetGuard, - fallback: Option>, -} - -// arbitrary choice to minimize changing all of veloren -pub type Localization = LocalizationGuard; - -impl LocalizationGuard { - /// Get a localized text from the given key - /// - /// First lookup is done in the active language, second in - /// the fallback (if present). - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get<'a>(&'a self, key: &'a str) -> &str { - self.active.get(key).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get(key)) - .unwrap_or(key) - }) - } - - /// Get a variation of localized text from the given key - /// - /// `index` should be a random number from `0` to `u16::max()` - /// - /// If the key is not present in the localization object - /// then the key is returned. - pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { - self.active.get_variation(key, index).unwrap_or_else(|| { - self.fallback - .as_ref() - .and_then(|f| f.get_variation(key, index)) - .unwrap_or(key) - }) - } - - /// Return the missing keys compared to the reference language - fn list_missing_entries(&self) -> (HashSet, HashSet) { - if let Some(ref_lang) = &self.fallback { - let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); - let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); - let strings = reference_string_keys - .difference(&string_keys) - .cloned() - .collect(); - - let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); - let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); - let vectors = reference_vector_keys - .difference(&vector_keys) - .cloned() - .collect(); - - (strings, vectors) - } else { - (HashSet::default(), HashSet::default()) - } - } - - /// Log missing entries (compared to the reference language) as warnings - pub fn log_missing_entries(&self) { - let (missing_strings, missing_vectors) = self.list_missing_entries(); - for missing_key in missing_strings { - warn!( - "[{:?}] Missing string key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - for missing_key in missing_vectors { - warn!( - "[{:?}] Missing vector key {:?}", - self.metadata().language_identifier, - missing_key - ); - } - } - - pub fn fonts(&self) -> &Fonts { &self.active.fonts } - - pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } -} - -impl LocalizationHandle { - pub fn set_english_fallback(&mut self, use_english_fallback: bool) { - self.use_english_fallback = use_english_fallback; - } - - pub fn read(&self) -> LocalizationGuard { - LocalizationGuard { - active: self.active.read(), - fallback: if self.use_english_fallback { - self.fallback.map(|f| f.read()) - } else { - None - }, - } - } - - pub fn load(specifier: &str) -> Result { - let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); - let language_key = ["voxygen.i18n.", specifier].concat(); - let is_default = language_key == default_key; - Ok(Self { - active: Language::load(&language_key)?, - fallback: if is_default { - None - } else { - Language::load(&default_key).ok() - }, - use_english_fallback: false, - }) - } - - pub fn load_expect(specifier: &str) -> Self { - Self::load(specifier).expect("Can't load language files") - } - - pub fn reloaded(&mut self) -> bool { self.active.reloaded() } -} - -struct FindManifests; - -impl assets::Compound for FindManifests { - fn load(_: &assets::AssetCache, _: &str) -> Result { - Ok(Self) - } -} - -impl assets::DirLoadable for FindManifests { - fn select_ids( - source: &S, - specifier: &str, - ) -> io::Result> { - let mut specifiers = Vec::new(); - - source.read_dir(specifier, &mut |entry| { - if let DirEntry::Directory(spec) = entry { - let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); - if source.exists(DirEntry::File(&manifest_spec, "ron")) { - specifiers.push(manifest_spec.into()); - } - } - })?; - - Ok(specifiers) - } -} - -#[derive(Clone, Debug)] -struct LocalizationList(Vec); - -impl assets::Compound for LocalizationList { - fn load( - cache: &assets::AssetCache, - specifier: &str, - ) -> Result { - // List language directories - let languages = assets::load_dir::(specifier, false) - .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) - .ids() - .filter_map(|spec| cache.load::(spec).ok()) - .map(|localization| localization.read().metadata.clone()) - .collect(); - - Ok(LocalizationList(languages)) - } -} - -/// Load all the available languages located in the voxygen asset directory -pub fn list_localizations() -> Vec { - LocalizationList::load_expect_cloned("voxygen.i18n").0 -} - -/// List localization directories as a `PathBuf` vector -pub fn i18n_directories(i18n_dir: &Path) -> Vec { - fs::read_dir(i18n_dir) - .unwrap() - .map(|res| res.map(|e| e.path()).unwrap()) - .filter(|e| e.is_dir()) - .collect() -} - -#[cfg(test)] -mod tests { - use super::assets; - // Test that localization list is loaded (not empty) - #[test] - fn test_localization_list() { - let list = super::list_localizations(); - assert!(!list.is_empty()); - } - - // Test that reference language can be loaded - #[test] - fn test_localization_handle() { - let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); - } - - // Test to verify all languages that they are VALID and loadable, without - // need of git just on the local assets folder - #[test] - fn verify_all_localizations() { - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::verification::verify_all_localizations(&root_dir, i18n_asset_path); - } - - // Test to verify all languages and print missing and faulty localisation - #[test] - #[ignore] - fn test_all_localizations() { - // Options - let be_verbose = true; - // Generate paths - let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/"); - let root_dir = assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&root_dir, i18n_asset_path, be_verbose); - } -} diff --git a/voxygen/i18n/src/gitfragments.rs b/voxygen/i18n/src/gitfragments.rs new file mode 100644 index 0000000000..ec77568837 --- /dev/null +++ b/voxygen/i18n/src/gitfragments.rs @@ -0,0 +1,157 @@ +//! fragment attached with git versioning information +use crate::raw::RawFragment; +use hashbrown::HashMap; +use std::path::Path; + +#[derive(Copy, Clone, Eq, Hash, Debug, PartialEq)] +pub(crate) enum LocalizationState { + UpToDate, + NotFound, + Outdated, + Unused, +} + +pub(crate) const ALL_LOCALIZATION_STATES: [Option; 5] = [ + Some(LocalizationState::UpToDate), + Some(LocalizationState::NotFound), + Some(LocalizationState::Outdated), + Some(LocalizationState::Unused), + None, +]; + +#[derive(Clone, Debug)] +pub(crate) struct LocalizationEntryState { + pub(crate) key_line: Option, + pub(crate) chuck_line_range: Option<(usize, usize)>, + pub(crate) commit_id: Option, + pub(crate) state: Option, +} + +impl LocalizationState { + pub(crate) fn print(this: &Option) -> String { + match this { + Some(LocalizationState::UpToDate) => "UpToDate", + Some(LocalizationState::NotFound) => "NotFound", + Some(LocalizationState::Outdated) => "Outdated", + Some(LocalizationState::Unused) => "Unused", + None => "Unknown", + } + .to_owned() + } +} + +impl LocalizationEntryState { + fn new(key_line: Option) -> LocalizationEntryState { + LocalizationEntryState { + key_line, + chuck_line_range: None, + commit_id: None, + state: None, + } + } +} + +/// Returns the Git blob associated with the given reference and path +pub(crate) fn read_file_from_path<'a>( + repo: &'a git2::Repository, + reference: &git2::Reference, + path: &std::path::Path, +) -> git2::Blob<'a> { + let tree = reference + .peel_to_tree() + .expect("Impossible to peel HEAD to a tree object"); + tree.get_path(path) + .unwrap_or_else(|_| { + panic!( + "Impossible to find the file {:?} in reference {:?}", + path, + reference.name() + ) + }) + .to_object(repo) + .unwrap() + .peel_to_blob() + .expect("Impossible to fetch the Git object") +} + +/// Extend a Fragment with historical git data +/// The actual translation gets dropped +/// TODO: transform vector_map too +pub(crate) fn transform_fragment<'a>( + repo: &'a git2::Repository, + fragment: (&Path, RawFragment), + file_blob: &git2::Blob, +) -> RawFragment { + let (path, fragment) = fragment; + // Find key start lines by searching all lines which have `:` in them (as they + // are probably keys) and getting the first part of such line trimming + // whitespace and quotes. Quite buggy heuristic + let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file"); + // we only need the key part of the file to process + let file_content_keys = file_content.lines().enumerate().filter_map(|(no, line)| { + line.split_once(':').map(|(key, _)| { + let mut key = key.trim().chars(); + key.next(); + key.next_back(); + (no, key.as_str()) + }) + }); + //speed up the search by sorting all keys! + let mut file_content_keys_sorted = file_content_keys.into_iter().collect::>(); + file_content_keys_sorted.sort_by_key(|(_, key)| *key); + + let mut result = RawFragment:: { + string_map: HashMap::new(), + vector_map: HashMap::new(), + }; + + for (original_key, _) in fragment.string_map { + let line_nb = file_content_keys_sorted + .binary_search_by_key(&original_key.as_str(), |(_, key)| *key) + .map_or_else( + |_| { + eprintln!( + "Key {} does not have a git line in it's state!", + original_key + ); + None + }, + |id| Some(file_content_keys_sorted[id].0), + ); + + result + .string_map + .insert(original_key, LocalizationEntryState::new(line_nb)); + } + + // Find commit for each keys, THIS PART IS SLOW (2s/4s) + for e in repo + .blame_file(path, None) + .expect("Impossible to generate the Git blame") + .iter() + { + for (_, state) in result.string_map.iter_mut() { + if let Some(line) = state.key_line { + let range = ( + e.final_start_line(), + e.final_start_line() + e.lines_in_hunk(), + ); + if line + 1 >= range.0 && line + 1 < range.1 { + state.chuck_line_range = Some(range); + state.commit_id = state.commit_id.map_or_else( + || Some(e.final_commit_id()), + |existing_commit| match repo + .graph_descendant_of(e.final_commit_id(), existing_commit) + { + Ok(true) => Some(e.final_commit_id()), + Ok(false) => Some(existing_commit), + Err(err) => panic!("{}", err), + }, + ); + } + } + } + } + + result +} diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 1d8462e15d..160a5118c0 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -1,7 +1,362 @@ #[cfg(any(feature = "bin", test))] pub mod analysis; -mod data; +#[cfg(any(feature = "bin", test))] +mod gitfragments; +mod path; +mod raw; +#[cfg(any(feature = "bin", test))] pub mod stats; pub mod verification; -use common_assets as assets; -pub use data::*; +//reexport +pub use path::BasePath; + +use crate::path::{LANG_EXTENSION, LANG_MANIFEST_FILE}; +use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle}; +use hashbrown::{HashMap, HashSet}; +use raw::{RawFragment, RawLanguage, RawManifest}; +use serde::{Deserialize, Serialize}; +use std::{io, path::PathBuf}; +use tracing::warn; + +/// The reference language, aka the more up-to-date localization data. +/// Also the default language at first startup. +pub const REFERENCE_LANG: &str = "en"; + +/// How a language can be described +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LanguageMetadata { + /// A human friendly language name (e.g. "English (US)") + pub language_name: String, + + /// A short text identifier for this language (e.g. "en_US") + /// + /// On the opposite of `language_name` that can change freely, + /// `language_identifier` value shall be stable in time as it + /// is used by setting components to store the language + /// selected by the user. + pub language_identifier: String, +} + +/// Store font metadata +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Font { + /// Key to retrieve the font in the asset system + pub asset_key: String, + + /// Scale ratio to resize the UI text dynamically + scale_ratio: f32, +} + +impl Font { + /// Scale input size to final UI size + pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 } +} + +/// Store font metadata +pub type Fonts = HashMap; + +/// Store internationalization data +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Language { + /// A map storing the localized texts + /// + /// Localized content can be accessed using a String key. + pub(crate) string_map: HashMap, + + /// A map for storing variations of localized texts, for example multiple + /// ways of saying "Help, I'm under attack". Used primarily for npc + /// dialogue. + pub(crate) vector_map: HashMap>, + + /// Whether to convert the input text encoded in UTF-8 + /// into a ASCII version by using the `deunicode` crate. + pub(crate) convert_utf8_to_ascii: bool, + + /// Font configuration is stored here + pub(crate) fonts: Fonts, + + pub(crate) metadata: LanguageMetadata, +} + +impl Language { + /// Get a localized text from the given key + pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> { + self.string_map.get(key).map(String::as_str) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> { + self.vector_map.get(key).and_then(|v| { + if v.is_empty() { + None + } else { + Some(v[index as usize % v.len()].as_str()) + } + }) + } +} + +impl common_assets::Compound for Language { + fn load( + cache: &common_assets::AssetCache, + asset_key: &str, + ) -> Result { + let manifest = cache + .load::(&[asset_key, ".", LANG_MANIFEST_FILE].concat())? + .cloned(); + + // Walk through files in the folder, collecting localization fragment to merge + // inside the asked_localization + let mut fragments = HashMap::new(); + for fragment_asset in cache + .load_dir::>(asset_key, true)? + .iter() + { + let id = fragment_asset.id(); + // Activate this once ._manifest is fully transformed and only contains metadata + // or regex: ", + fallback: Option>, + pub use_english_fallback: bool, +} + +// RAII guard returned from Localization::read(), resembles AssetGuard +pub struct LocalizationGuard { + active: AssetGuard, + fallback: Option>, +} + +// arbitrary choice to minimize changing all of veloren +pub type Localization = LocalizationGuard; + +impl LocalizationGuard { + /// Get a localized text from the given key + /// + /// First lookup is done in the active language, second in + /// the fallback (if present). + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get<'a>(&'a self, key: &'a str) -> &str { + self.active.get(key).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get(key)) + .unwrap_or(key) + }) + } + + /// Get a variation of localized text from the given key + /// + /// `index` should be a random number from `0` to `u16::max()` + /// + /// If the key is not present in the localization object + /// then the key is returned. + pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str { + self.active.get_variation(key, index).unwrap_or_else(|| { + self.fallback + .as_ref() + .and_then(|f| f.get_variation(key, index)) + .unwrap_or(key) + }) + } + + /// Return the missing keys compared to the reference language + fn list_missing_entries(&self) -> (HashSet, HashSet) { + if let Some(ref_lang) = &self.fallback { + let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect(); + let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect(); + let strings = reference_string_keys + .difference(&string_keys) + .cloned() + .collect(); + + let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect(); + let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect(); + let vectors = reference_vector_keys + .difference(&vector_keys) + .cloned() + .collect(); + + (strings, vectors) + } else { + (HashSet::default(), HashSet::default()) + } + } + + /// Log missing entries (compared to the reference language) as warnings + pub fn log_missing_entries(&self) { + let (missing_strings, missing_vectors) = self.list_missing_entries(); + for missing_key in missing_strings { + warn!( + "[{:?}] Missing string key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + for missing_key in missing_vectors { + warn!( + "[{:?}] Missing vector key {:?}", + self.metadata().language_identifier, + missing_key + ); + } + } + + pub fn fonts(&self) -> &Fonts { &self.active.fonts } + + pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata } +} + +impl LocalizationHandle { + pub fn set_english_fallback(&mut self, use_english_fallback: bool) { + self.use_english_fallback = use_english_fallback; + } + + pub fn read(&self) -> LocalizationGuard { + LocalizationGuard { + active: self.active.read(), + fallback: if self.use_english_fallback { + self.fallback.map(|f| f.read()) + } else { + None + }, + } + } + + pub fn load(specifier: &str) -> Result { + let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat(); + let language_key = ["voxygen.i18n.", specifier].concat(); + let is_default = language_key == default_key; + Ok(Self { + active: Language::load(&language_key)?, + fallback: if is_default { + None + } else { + Language::load(&default_key).ok() + }, + use_english_fallback: false, + }) + } + + pub fn load_expect(specifier: &str) -> Self { + Self::load(specifier).expect("Can't load language files") + } + + pub fn reloaded(&mut self) -> bool { self.active.reloaded() } +} + +struct FindManifests; + +impl common_assets::Compound for FindManifests { + fn load( + _: &common_assets::AssetCache, + _: &str, + ) -> Result { + Ok(Self) + } +} + +impl common_assets::DirLoadable for FindManifests { + fn select_ids( + source: &S, + specifier: &str, + ) -> io::Result> { + let mut specifiers = Vec::new(); + + source.read_dir(specifier, &mut |entry| { + if let DirEntry::Directory(spec) = entry { + let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat(); + if source.exists(DirEntry::File(&manifest_spec, LANG_EXTENSION)) { + specifiers.push(manifest_spec.into()); + } + } + })?; + + Ok(specifiers) + } +} + +#[derive(Clone, Debug)] +struct LocalizationList(Vec); + +impl common_assets::Compound for LocalizationList { + fn load( + cache: &common_assets::AssetCache, + specifier: &str, + ) -> Result { + // List language directories + let languages = common_assets::load_dir::(specifier, false) + .unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e)) + .ids() + .filter_map(|spec| cache.load::(spec).ok()) + .map(|localization| localization.read().metadata.clone()) + .collect(); + + Ok(LocalizationList(languages)) + } +} + +/// Load all the available languages located in the voxygen asset directory +pub fn list_localizations() -> Vec { + LocalizationList::load_expect_cloned("voxygen.i18n").0 +} + +#[cfg(test)] +mod tests { + use crate::path::BasePath; + + // Test that localization list is loaded (not empty) + #[test] + fn test_localization_list() { + let list = super::list_localizations(); + assert!(!list.is_empty()); + } + + // Test that reference language can be loaded + #[test] + fn test_localization_handle() { + let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG); + } + + // Test to verify all languages that they are VALID and loadable, without + // need of git just on the local assets folder + #[test] + fn verify_all_localizations() { + // Generate paths + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::verification::verify_all_localizations(&BasePath::new(&root_dir)); + } + + // Test to verify all languages and print missing and faulty localisation + #[test] + #[ignore] + fn test_all_localizations() { + // Generate paths + let root_dir = common_assets::find_root().expect("Failed to discover repository root"); + crate::analysis::test_all_localizations(&BasePath::new(&root_dir), true, false); + } +} diff --git a/voxygen/i18n/src/path.rs b/voxygen/i18n/src/path.rs new file mode 100644 index 0000000000..a8f961eef5 --- /dev/null +++ b/voxygen/i18n/src/path.rs @@ -0,0 +1,138 @@ +use std::path::{Path, PathBuf}; + +pub(crate) const LANG_MANIFEST_FILE: &str = "_manifest"; +pub(crate) const LANG_EXTENSION: &str = "ron"; + +#[derive(Clone)] +pub struct BasePath { + ///repo part, git main folder + root_path: PathBuf, + ///relative path to i18n path which contains, currently + /// 'assets/voxygen/i18n' + relative_i18n_root_path: PathBuf, + ///i18n_root_folder + cache: PathBuf, +} + +impl BasePath { + pub fn new(root_path: &Path) -> Self { + let relative_i18n_root_path = Path::new("assets/voxygen/i18n").to_path_buf(); + let cache = root_path.join(&relative_i18n_root_path); + assert!( + cache.is_dir(), + "i18n_root_path folder doesn't exist, something is wrong!" + ); + Self { + root_path: root_path.to_path_buf(), + relative_i18n_root_path, + cache, + } + } + + pub fn root_path(&self) -> &Path { &self.root_path } + + pub fn relative_i18n_root_path(&self) -> &Path { &self.relative_i18n_root_path } + + /// absolute path to `relative_i18n_root_path` + pub fn i18n_root_path(&self) -> &Path { &self.cache } + + pub fn i18n_path(&self, language_identifier: &str) -> LangPath { + LangPath::new(self, language_identifier) + } + + /// List localization directories + pub fn i18n_directories(&self) -> Vec { + std::fs::read_dir(&self.cache) + .unwrap() + .map(|res| res.unwrap()) + .filter(|e| e.file_type().unwrap().is_dir()) + .map(|e| LangPath::new(self, e.file_name().to_str().unwrap())) + .collect() + } +} + +impl core::fmt::Debug for BasePath { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", &self.cache) + } +} + +#[derive(Clone)] +pub struct LangPath { + base: BasePath, + /// `en`, `de_DE`, `fr_FR`, etc.. + language_identifier: String, + /// i18n_path + cache: PathBuf, +} + +impl LangPath { + fn new(base: &BasePath, language_identifier: &str) -> Self { + let cache = base.i18n_root_path().join(language_identifier); + if !cache.is_dir() { + panic!("language folder '{}' doesn't exist", language_identifier); + } + Self { + base: base.clone(), + language_identifier: language_identifier.to_owned(), + cache, + } + } + + pub fn base(&self) -> &BasePath { &self.base } + + pub fn language_identifier(&self) -> &str { &self.language_identifier } + + ///absolute path to `i18n_root_path` + `language_identifier` + pub fn i18n_path(&self) -> &Path { &self.cache } + + /// fragment or manifest file, based on a path + pub fn sub_path(&self, sub_path: &Path) -> PathBuf { self.cache.join(sub_path) } + + /// fragment or manifest file, based on a string without extension + pub fn file(&self, name_without_extension: &str) -> PathBuf { + self.cache + .join(format!("{}.{}", name_without_extension, LANG_EXTENSION)) + } + + /// return all fragments sub_pathes + pub(crate) fn fragments(&self) -> Result, std::io::Error> { + let mut result = vec![]; + recursive_fragments_paths_in_language(self, Path::new(""), &mut result)?; + Ok(result) + } +} + +//unwraps cant fail as they are in same Path +fn recursive_fragments_paths_in_language( + lpath: &LangPath, + subfolder: &Path, + result: &mut Vec, +) -> Result<(), std::io::Error> { + let search_dir = lpath.sub_path(subfolder); + for fragment_file in search_dir.read_dir()?.flatten() { + let file_type = fragment_file.file_type()?; + let full_path = fragment_file.path(); + let relative_path = full_path.strip_prefix(lpath.i18n_path()).unwrap(); + if file_type.is_dir() { + recursive_fragments_paths_in_language(lpath, relative_path, result)?; + } else if file_type.is_file() + && relative_path != Path::new(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)) + { + result.push(relative_path.to_path_buf()); + } + } + Ok(()) +} + +impl core::fmt::Debug for LangPath { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{:?}", + self.base + .relative_i18n_root_path + .join(&self.language_identifier) + ) + } +} diff --git a/voxygen/i18n/src/raw.rs b/voxygen/i18n/src/raw.rs new file mode 100644 index 0000000000..56e4e4b1c7 --- /dev/null +++ b/voxygen/i18n/src/raw.rs @@ -0,0 +1,132 @@ +//! handle the loading of a `Language` +use crate::{ + path::{LangPath, LANG_EXTENSION, LANG_MANIFEST_FILE}, + Fonts, Language, LanguageMetadata, +}; +use deunicode::deunicode; +use hashbrown::hash_map::HashMap; +use ron::de::from_reader; +use serde::{Deserialize, Serialize}; +use std::{fs, path::PathBuf}; + +/// Raw localization metadata from LANG_MANIFEST_FILE file +/// See `Language` for more info on each attributes +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawManifest { + pub(crate) convert_utf8_to_ascii: bool, + pub(crate) fonts: Fonts, + pub(crate) metadata: LanguageMetadata, +} + +/// Raw localization data from one specific file +/// These structs are meant to be merged into a Language +#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +pub(crate) struct RawFragment { + pub(crate) string_map: HashMap, + pub(crate) vector_map: HashMap>, +} + +pub(crate) struct RawLanguage { + pub(crate) manifest: RawManifest, + pub(crate) fragments: HashMap>, +} + +#[derive(Debug)] +pub(crate) enum RawError { + RonError(ron::Error), +} + +pub(crate) fn load_manifest(path: &LangPath) -> Result { + let manifest_file = path.file(LANG_MANIFEST_FILE); + tracing::debug!(?manifest_file, "manifest loading"); + let f = fs::File::open(&manifest_file)?; + let manifest: RawManifest = from_reader(f).map_err(RawError::RonError)?; + // verify that the folder name `de_DE` matches the value inside the metadata! + assert_eq!( + manifest.metadata.language_identifier, + path.language_identifier() + ); + Ok(manifest) +} + +pub(crate) fn load_raw_language( + path: &LangPath, + manifest: RawManifest, +) -> Result, common_assets::Error> { + //get List of files + let files = path.fragments()?; + + // Walk through each file in the directory + let mut fragments = HashMap::new(); + for sub_path in files { + let f = fs::File::open(path.sub_path(&sub_path))?; + let fragment = from_reader(f).map_err(RawError::RonError)?; + fragments.insert(sub_path, fragment); + } + + Ok(RawLanguage { + manifest, + fragments, + }) +} + +impl From> for Language { + fn from(raw: RawLanguage) -> Self { + let mut string_map = HashMap::new(); + let mut vector_map = HashMap::new(); + + for (_, fragment) in raw.fragments { + string_map.extend(fragment.string_map); + vector_map.extend(fragment.vector_map); + } + + let convert_utf8_to_ascii = raw.manifest.convert_utf8_to_ascii; + + // Update the text if UTF-8 to ASCII conversion is enabled + if convert_utf8_to_ascii { + for value in string_map.values_mut() { + *value = deunicode(value); + } + + for value in vector_map.values_mut() { + *value = value.iter().map(|s| deunicode(s)).collect(); + } + } + let mut metadata = raw.manifest.metadata; + metadata.language_name = deunicode(&metadata.language_name); + + Self { + string_map, + vector_map, + convert_utf8_to_ascii, + fonts: raw.manifest.fonts, + metadata, + } + } +} + +impl core::fmt::Display for RawError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + RawError::RonError(e) => write!(f, "{}", e), + } + } +} + +impl std::error::Error for RawError {} + +impl From for common_assets::Error { + fn from(e: RawError) -> Self { Self::Conversion(Box::new(e)) } +} + +impl common_assets::Asset for RawManifest { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} + +impl common_assets::Asset for RawFragment { + type Loader = common_assets::RonLoader; + + const EXTENSION: &'static str = LANG_EXTENSION; +} diff --git a/voxygen/i18n/src/stats.rs b/voxygen/i18n/src/stats.rs new file mode 100644 index 0000000000..9ab93fd062 --- /dev/null +++ b/voxygen/i18n/src/stats.rs @@ -0,0 +1,203 @@ +use crate::{ + gitfragments::{LocalizationEntryState, LocalizationState, ALL_LOCALIZATION_STATES}, + raw::RawLanguage, +}; +use hashbrown::HashMap; +use std::path::PathBuf; + +#[derive(Default, Debug, PartialEq)] +pub(crate) struct LocalizationStats { + pub(crate) uptodate_entries: usize, + pub(crate) notfound_entries: usize, + pub(crate) unused_entries: usize, + pub(crate) outdated_entries: usize, + pub(crate) errors: usize, +} + +#[allow(clippy::type_complexity)] +pub(crate) struct LocalizationAnalysis { + language_identifier: String, + pub(crate) data: HashMap, Vec<(PathBuf, String, Option)>>, +} + +impl LocalizationStats { + /// Calculate key count that actually matter for the status of the + /// translation Unused entries don't break the game + pub(crate) fn get_real_entry_count(&self) -> usize { + self.outdated_entries + self.notfound_entries + self.errors + self.uptodate_entries + } +} + +impl LocalizationAnalysis { + pub(crate) fn new(language_identifier: &str) -> Self { + let mut data = HashMap::new(); + data.insert(Some(LocalizationState::UpToDate), vec![]); + data.insert(Some(LocalizationState::NotFound), vec![]); + data.insert(Some(LocalizationState::Unused), vec![]); + data.insert(Some(LocalizationState::Outdated), vec![]); + data.insert(None, vec![]); + Self { + language_identifier: language_identifier.to_owned(), + data, + } + } + + fn show( + &self, + state: Option, + ref_language: &RawLanguage, + be_verbose: bool, + ) { + let entries = self.data.get(&state).unwrap_or_else(|| { + panic!( + "called on invalid state: {}", + LocalizationState::print(&state) + ) + }); + if entries.is_empty() { + return; + } + println!("\n\t[{}]", LocalizationState::print(&state)); + for (path, key, commit_id) in entries { + if be_verbose { + let our_commit = LocalizationAnalysis::print_commit(commit_id); + let ref_commit = ref_language + .fragments + .get(path) + .and_then(|entry| entry.string_map.get(key)) + .and_then(|s| s.commit_id) + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()); + println!("{:60}| {:40} | {:40}", key, our_commit, ref_commit,); + } else { + println!("{}", key); + } + } + } + + fn csv(&self, state: Option) { + let entries = self + .data + .get(&state) + .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); + for (path, key, commit_id) in entries { + let our_commit = LocalizationAnalysis::print_commit(commit_id); + println!( + "{},{:?},{},{},{}", + self.language_identifier, + path, + key, + LocalizationState::print(&state), + our_commit + ); + } + } + + fn print_commit(commit_id: &Option) -> String { + commit_id + .map(|s| format!("{}", s)) + .unwrap_or_else(|| "None".to_owned()) + } +} + +pub(crate) fn print_translation_stats( + language_identifier: &str, + ref_language: &RawLanguage, + stats: &LocalizationStats, + state_map: &LocalizationAnalysis, + be_verbose: bool, +) { + let real_entry_count = stats.get_real_entry_count() as f32; + let uptodate_percent = (stats.uptodate_entries as f32 / real_entry_count) * 100_f32; + let outdated_percent = (stats.outdated_entries as f32 / real_entry_count) * 100_f32; + let untranslated_percent = ((stats.errors + stats.errors) as f32 / real_entry_count) * 100_f32; + + // Display + if be_verbose { + println!( + "\n{:60}| {:40} | {:40}", + "Key name", language_identifier, ref_language.manifest.metadata.language_identifier, + ); + } else { + println!("\nKey name"); + } + + for state in &ALL_LOCALIZATION_STATES { + if state == &Some(LocalizationState::UpToDate) { + continue; + } + state_map.show(*state, ref_language, be_verbose); + } + + println!( + "\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries", + stats.uptodate_entries, + stats.outdated_entries, + stats.unused_entries, + stats.notfound_entries, + stats.errors, + ); + + println!( + "{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n", + uptodate_percent, outdated_percent, untranslated_percent, + ); +} + +pub(crate) fn print_csv_file(state_map: &LocalizationAnalysis) { + println!("country_code,file_name,translation_code,status,git_commit"); + + for state in &ALL_LOCALIZATION_STATES { + if state == &Some(LocalizationState::UpToDate) { + continue; + } + state_map.csv(*state); + } +} + +pub(crate) fn print_overall_stats( + analysis: HashMap, +) { + let mut overall_uptodate_entry_count = 0; + let mut overall_outdated_entry_count = 0; + let mut overall_untranslated_entry_count = 0; + let mut overall_real_entry_count = 0; + + println!("-----------------------------------------------------------------------------"); + println!("Overall Translation Status"); + println!("-----------------------------------------------------------------------------"); + println!( + "{:12}| {:8} | {:8} | {:8} | {:8} | {:8}", + "", "up-to-date", "outdated", "untranslated", "unused", "errors", + ); + + let mut i18n_stats: Vec<(&String, &(_, LocalizationStats))> = analysis.iter().collect(); + i18n_stats.sort_by_key(|(_, (_, v))| v.notfound_entries); + + for (path, (_, test_result)) in i18n_stats { + let LocalizationStats { + uptodate_entries: uptodate, + outdated_entries: outdated, + unused_entries: unused, + notfound_entries: untranslated, + errors, + } = test_result; + overall_uptodate_entry_count += uptodate; + overall_outdated_entry_count += outdated; + overall_untranslated_entry_count += untranslated; + overall_real_entry_count += test_result.get_real_entry_count(); + + println!( + "{:12}|{:8} |{:6} |{:8} |{:6} |{:8}", + path, uptodate, outdated, untranslated, unused, errors, + ); + } + + println!( + "\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated", + (overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + (overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32, + ); + println!("-----------------------------------------------------------------------------\n"); +} diff --git a/voxygen/i18n/src/verification.rs b/voxygen/i18n/src/verification.rs index ad7897c386..fa784b9db5 100644 --- a/voxygen/i18n/src/verification.rs +++ b/voxygen/i18n/src/verification.rs @@ -1,52 +1,17 @@ -use ron::de::from_reader; -use std::{fs, path::Path}; +use crate::path::{BasePath, LangPath, LANG_MANIFEST_FILE}; -use crate::data::{i18n_directories, LocalizationFragment, LANG_MANIFEST_FILE, REFERENCE_LANG}; - -fn verify_localization_directory(root_dir: &Path, directory_path: &Path) { - // Walk through each file in the directory - for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() { - if let Ok(file_type) = i18n_file.file_type() { - // Skip folders and the manifest file (which does not contain the same struct we - // want to load) - if file_type.is_file() { - let full_path = i18n_file.path(); - println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap()); - let f = fs::File::open(&full_path).expect("Failed opening file"); - let _loc: LocalizationFragment = match from_reader(f) { - Ok(v) => v, - Err(e) => { - panic!( - "Could not parse {} RON file, error: {}", - full_path.to_string_lossy(), - e - ); - }, - }; - } else if file_type.is_dir() { - verify_localization_directory(root_dir, &i18n_file.path()); - } - } - } -} +use crate::{raw, REFERENCE_LANG}; /// Test to verify all languages that they are VALID and loadable, without /// need of git just on the local assets folder -/// `root_dir` - absolute path to main repo -/// `asset_path` - relative path to asset directory (right now it is -/// 'assets/voxygen/i18n') -pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { - let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG); - let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron"); +pub fn verify_all_localizations(path: &BasePath) { + let ref_i18n_path = path.i18n_path(REFERENCE_LANG); + let ref_i18n_manifest_path = ref_i18n_path.file(LANG_MANIFEST_FILE); assert!( - root_dir.join(&ref_i18n_dir_path).is_dir(), - "Reference language folder doesn't exist, something is wrong!" - ); - assert!( - root_dir.join(&ref_i18n_path).is_file(), + ref_i18n_manifest_path.is_file(), "Reference language manifest file doesn't exist, something is wrong!" ); - let i18n_directories = i18n_directories(&root_dir.join(asset_path)); + let i18n_directories = path.i18n_directories(); // This simple check ONLY guarantees that an arbitrary minimum of translation // files exists. It's just to notice unintentional deletion of all // files, or modifying the paths. In case you want to delete all @@ -57,11 +22,13 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) { folder is empty?" ); for i18n_directory in i18n_directories { - println!( - "verifying {:?}", - i18n_directory.strip_prefix(&root_dir).unwrap() - ); + println!("verifying {:?}", i18n_directory); // Walk through each files and try to load them - verify_localization_directory(root_dir, &i18n_directory); + verify_localization_directory(&i18n_directory); } } + +fn verify_localization_directory(path: &LangPath) { + let manifest = raw::load_manifest(path).expect("error accessing manifest file"); + raw::load_raw_language(path, manifest).expect("error accessing fragment file"); +} diff --git a/voxygen/src/hud/bag.rs b/voxygen/src/hud/bag.rs index 343df2f232..4a21b416db 100644 --- a/voxygen/src/hud/bag.rs +++ b/voxygen/src/hud/bag.rs @@ -7,7 +7,6 @@ use super::{ }; use crate::{ game_input::GameInput, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -31,6 +30,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, State as ConrodState, Text}, widget_ids, Color, Colorable, Positionable, Scalar, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use crate::hud::slots::SlotKind; use specs::Entity as EcsEntity; diff --git a/voxygen/src/hud/buffs.rs b/voxygen/src/hud/buffs.rs index 5fc82242df..393dd06fc0 100644 --- a/voxygen/src/hud/buffs.rs +++ b/voxygen/src/hud/buffs.rs @@ -4,10 +4,10 @@ use super::{ }; use crate::{ hud::{self, BuffPosition}, - i18n::Localization, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, }; +use i18n::Localization; use common::comp::{BuffKind, Buffs, Energy, Health}; use conrod_core::{ diff --git a/voxygen/src/hud/buttons.rs b/voxygen/src/hud/buttons.rs index 79bde0effd..d046c8a783 100644 --- a/voxygen/src/hud/buttons.rs +++ b/voxygen/src/hud/buttons.rs @@ -4,7 +4,6 @@ use super::{ }; use crate::{ game_input::GameInput, - i18n::Localization, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, window::KeyMouse, GlobalState, @@ -15,6 +14,7 @@ use conrod_core::{ widget::{self, Button, Image, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { bag, diff --git a/voxygen/src/hud/chat.rs b/voxygen/src/hud/chat.rs index c05177e1ca..8a48c212e7 100644 --- a/voxygen/src/hud/chat.rs +++ b/voxygen/src/hud/chat.rs @@ -2,7 +2,7 @@ use super::{ img_ids::Imgs, ChatTab, ERROR_COLOR, FACTION_COLOR, GROUP_COLOR, INFO_COLOR, KILL_COLOR, OFFLINE_COLOR, ONLINE_COLOR, REGION_COLOR, SAY_COLOR, TELL_COLOR, TEXT_COLOR, WORLD_COLOR, }; -use crate::{i18n::Localization, settings::chat::MAX_CHAT_TABS, ui::fonts::Fonts, GlobalState}; +use crate::{settings::chat::MAX_CHAT_TABS, ui::fonts::Fonts, GlobalState}; use client::{cmd, Client}; use common::comp::{ chat::{KillSource, KillType}, @@ -22,6 +22,7 @@ use conrod_core::{ widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Ui, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use std::collections::{HashSet, VecDeque}; widget_ids! { diff --git a/voxygen/src/hud/crafting.rs b/voxygen/src/hud/crafting.rs index 79093537ac..dd2e917b9c 100644 --- a/voxygen/src/hud/crafting.rs +++ b/voxygen/src/hud/crafting.rs @@ -4,12 +4,9 @@ use super::{ item_imgs::{animate_by_pulse, ItemImgs, ItemKey::Tool}, Show, TEXT_COLOR, TEXT_DULL_RED_COLOR, TEXT_GRAY_COLOR, UI_HIGHLIGHT_0, UI_MAIN, }; -use crate::{ - i18n::Localization, - ui::{ - fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable, Tooltip, - TooltipManager, Tooltipable, - }, +use crate::ui::{ + fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable, Tooltip, + TooltipManager, Tooltipable, }; use client::{self, Client}; use common::{ @@ -29,6 +26,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text, TextEdit}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::sync::Arc; use strum::IntoEnumIterator; diff --git a/voxygen/src/hud/diary.rs b/voxygen/src/hud/diary.rs index 8112d784d1..da5af98b50 100644 --- a/voxygen/src/hud/diary.rs +++ b/voxygen/src/hud/diary.rs @@ -3,16 +3,14 @@ use super::{ item_imgs::{animate_by_pulse, ItemImgs, ItemKey::Tool}, Show, CRITICAL_HP_COLOR, HP_COLOR, TEXT_COLOR, UI_HIGHLIGHT_0, UI_MAIN, XP_COLOR, }; -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}; use conrod_core::{ color, image::Id, widget::{self, button, Button, Image, Rectangle, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use client::{self, Client}; use common::comp::{ diff --git a/voxygen/src/hud/esc_menu.rs b/voxygen/src/hud/esc_menu.rs index 5573cf3ad6..92f989ad68 100644 --- a/voxygen/src/hud/esc_menu.rs +++ b/voxygen/src/hud/esc_menu.rs @@ -1,9 +1,10 @@ use super::{img_ids::Imgs, settings_window::SettingsTab, TEXT_COLOR}; -use crate::{i18n::Localization, ui::fonts::Fonts}; +use crate::ui::fonts::Fonts; use conrod_core::{ widget::{self, Button, Image}, widget_ids, Color, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/group.rs b/voxygen/src/hud/group.rs index 74ee3ed8b2..71fb4a5f56 100644 --- a/voxygen/src/hud/group.rs +++ b/voxygen/src/hud/group.rs @@ -9,7 +9,6 @@ use super::{ use crate::{ game_input::GameInput, hud, - i18n::Localization, settings::Settings, ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, @@ -27,6 +26,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use specs::{saveload::MarkerAllocator, WorldExt}; widget_ids! { diff --git a/voxygen/src/hud/loot_scroller.rs b/voxygen/src/hud/loot_scroller.rs index 676291465b..53e88d6820 100644 --- a/voxygen/src/hud/loot_scroller.rs +++ b/voxygen/src/hud/loot_scroller.rs @@ -4,10 +4,7 @@ use super::{ item_imgs::ItemImgs, Show, Windows, TEXT_COLOR, }; -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, ItemTooltip, ItemTooltipManager, ItemTooltipable}; use client::Client; use common::comp::inventory::item::{ItemDef, MaterialStatManifest, Quality}; use conrod_core::{ @@ -16,6 +13,7 @@ use conrod_core::{ widget::{self, Image, List, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::{collections::VecDeque, sync::Arc}; widget_ids! { diff --git a/voxygen/src/hud/map.rs b/voxygen/src/hud/map.rs index 3742f2565c..9039add9b4 100644 --- a/voxygen/src/hud/map.rs +++ b/voxygen/src/hud/map.rs @@ -4,7 +4,6 @@ use super::{ TEXT_BG, TEXT_BLUE_COLOR, TEXT_COLOR, TEXT_GRAY_COLOR, TEXT_VELORITE, UI_HIGHLIGHT_0, UI_MAIN, }; use crate::{ - i18n::Localization, session::settings_change::{Interface as InterfaceChange, Interface::*}, ui::{fonts::Fonts, img_ids, ImageFrame, Tooltip, TooltipManager, Tooltipable}, GlobalState, @@ -17,6 +16,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use specs::{saveload::MarkerAllocator, WorldExt}; use vek::*; diff --git a/voxygen/src/hud/mod.rs b/voxygen/src/hud/mod.rs index af9304279d..4a3b4693ba 100644 --- a/voxygen/src/hud/mod.rs +++ b/voxygen/src/hud/mod.rs @@ -55,7 +55,6 @@ use crate::{ ecs::{comp as vcomp, comp::HpFloaterList}, game_input::GameInput, hud::{img_ids::ImgsRot, prompt_dialog::DialogOutcomeEvent}, - i18n::Localization, render::UiDrawer, scene::camera::{self, Camera}, session::{ @@ -100,6 +99,7 @@ use conrod_core::{ widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, }; use hashbrown::{HashMap, HashSet}; +use i18n::Localization; use rand::Rng; use specs::{Entity as EcsEntity, Join, WorldExt}; use std::{ diff --git a/voxygen/src/hud/overhead.rs b/voxygen/src/hud/overhead.rs index 4d58d7aef0..c0c191e50d 100644 --- a/voxygen/src/hud/overhead.rs +++ b/voxygen/src/hud/overhead.rs @@ -5,7 +5,6 @@ use super::{ }; use crate::{ hud::{get_buff_image, get_buff_info}, - i18n::Localization, settings::InterfaceSettings, ui::{fonts::Fonts, Ingameable}, }; @@ -16,6 +15,7 @@ use conrod_core::{ widget::{self, Image, Rectangle, Text}, widget_ids, Color, Colorable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; const MAX_BUBBLE_WIDTH: f64 = 250.0; widget_ids! { diff --git a/voxygen/src/hud/overitem.rs b/voxygen/src/hud/overitem.rs index 368350389b..7522e04f06 100644 --- a/voxygen/src/hud/overitem.rs +++ b/voxygen/src/hud/overitem.rs @@ -1,6 +1,5 @@ use crate::{ game_input::GameInput, - i18n::Localization, settings::ControlSettings, ui::{fonts::Fonts, Ingameable}, }; @@ -9,6 +8,7 @@ use conrod_core::{ widget::{self, RoundedRectangle, Text}, widget_ids, Color, Colorable, Positionable, Widget, WidgetCommon, }; +use i18n::Localization; use std::borrow::Cow; use keyboard_keynames::key_layout::KeyLayout; diff --git a/voxygen/src/hud/popup.rs b/voxygen/src/hud/popup.rs index 843bc57baf..a87feaecb2 100644 --- a/voxygen/src/hud/popup.rs +++ b/voxygen/src/hud/popup.rs @@ -1,11 +1,12 @@ use super::Show; -use crate::{i18n::Localization, ui::fonts::Fonts}; +use crate::ui::fonts::Fonts; use client::{self, Client}; use common_net::msg::Notification; use conrod_core::{ widget::{self, Text}, widget_ids, Color, Colorable, Positionable, Widget, WidgetCommon, }; +use i18n::Localization; use std::{collections::VecDeque, time::Instant}; widget_ids! { diff --git a/voxygen/src/hud/prompt_dialog.rs b/voxygen/src/hud/prompt_dialog.rs index 599eca1953..769100fc86 100644 --- a/voxygen/src/hud/prompt_dialog.rs +++ b/voxygen/src/hud/prompt_dialog.rs @@ -2,7 +2,6 @@ use super::{img_ids::Imgs, TEXT_COLOR, UI_HIGHLIGHT_0}; use crate::{ game_input::GameInput, hud::{Event, PromptDialogSettings}, - i18n::LocalizationHandle, settings::Settings, ui::fonts::Fonts, }; @@ -10,6 +9,7 @@ use conrod_core::{ widget::{self, Button, Image, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::LocalizationHandle; use keyboard_keynames::key_layout::KeyLayout; widget_ids! { diff --git a/voxygen/src/hud/settings_window/chat.rs b/voxygen/src/hud/settings_window/chat.rs index 60c540d8fd..6f74fcd20b 100644 --- a/voxygen/src/hud/settings_window/chat.rs +++ b/voxygen/src/hud/settings_window/chat.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, ChatTab, Show, TEXT_COLOR, TEXT_GRAY_COLOR, UI_HIGHLIGHT_0, UI_MAIN}, - i18n::Localization, session::settings_change::{Chat as ChatChange, Chat::*}, settings::chat::MAX_CHAT_TABS, ui::{fonts::Fonts, ImageSlider, ToggleButton}, @@ -14,6 +13,7 @@ use conrod_core::{ widget::{self, Button, DropDownList, Image, Rectangle, Text, TextEdit}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use std::cmp::Ordering; widget_ids! { diff --git a/voxygen/src/hud/settings_window/controls.rs b/voxygen/src/hud/settings_window/controls.rs index 51b842f936..005ba006b6 100644 --- a/voxygen/src/hud/settings_window/controls.rs +++ b/voxygen/src/hud/settings_window/controls.rs @@ -3,7 +3,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ game_input::GameInput, hud::{img_ids::Imgs, ERROR_COLOR, TEXT_BIND_CONFLICT_COLOR, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Control as ControlChange, Control::*}, ui::fonts::Fonts, GlobalState, @@ -14,6 +13,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Borderable, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use strum::IntoEnumIterator; widget_ids! { diff --git a/voxygen/src/hud/settings_window/gameplay.rs b/voxygen/src/hud/settings_window/gameplay.rs index bb059f528d..010c71739b 100644 --- a/voxygen/src/hud/settings_window/gameplay.rs +++ b/voxygen/src/hud/settings_window/gameplay.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, PressBehavior, MENU_BG, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Gameplay as GameplayChange, Gameplay::*}, ui::{fonts::Fonts, ImageSlider, ToggleButton}, GlobalState, @@ -13,6 +12,7 @@ use conrod_core::{ widget::{self, Button, DropDownList, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/interface.rs b/voxygen/src/hud/settings_window/interface.rs index f39def8fa6..27f4745a60 100644 --- a/voxygen/src/hud/settings_window/interface.rs +++ b/voxygen/src/hud/settings_window/interface.rs @@ -4,7 +4,6 @@ use crate::{ hud::{ img_ids::Imgs, BarNumbers, BuffPosition, CrosshairType, ShortcutNumbers, Show, TEXT_COLOR, }, - i18n::Localization, session::settings_change::{Interface as InterfaceChange, Interface::*}, ui::{fonts::Fonts, ImageSlider, ScaleMode, ToggleButton}, GlobalState, @@ -15,6 +14,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids{ diff --git a/voxygen/src/hud/settings_window/language.rs b/voxygen/src/hud/settings_window/language.rs index e1cb5787c7..5745b81bca 100644 --- a/voxygen/src/hud/settings_window/language.rs +++ b/voxygen/src/hud/settings_window/language.rs @@ -1,6 +1,5 @@ use crate::{ hud::{img_ids::Imgs, TEXT_COLOR}, - i18n::{list_localizations, Localization}, session::settings_change::{Language as LanguageChange, Language::*}, ui::{fonts::Fonts, ToggleButton}, GlobalState, @@ -10,6 +9,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::{list_localizations, Localization}; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/mod.rs b/voxygen/src/hud/settings_window/mod.rs index 73de93697a..e72101755f 100644 --- a/voxygen/src/hud/settings_window/mod.rs +++ b/voxygen/src/hud/settings_window/mod.rs @@ -8,7 +8,6 @@ mod video; use crate::{ hud::{img_ids::Imgs, Show, TEXT_COLOR, UI_HIGHLIGHT_0, UI_MAIN}, - i18n::Localization, session::settings_change::SettingsChange, ui::fonts::Fonts, GlobalState, @@ -18,6 +17,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use strum::IntoEnumIterator; use strum_macros::EnumIter; diff --git a/voxygen/src/hud/settings_window/sound.rs b/voxygen/src/hud/settings_window/sound.rs index f563b48f9e..ed655a0765 100644 --- a/voxygen/src/hud/settings_window/sound.rs +++ b/voxygen/src/hud/settings_window/sound.rs @@ -2,7 +2,6 @@ use super::{RESET_BUTTONS_HEIGHT, RESET_BUTTONS_WIDTH}; use crate::{ hud::{img_ids::Imgs, TEXT_COLOR}, - i18n::Localization, session::settings_change::{Audio as AudioChange, Audio::*}, ui::{fonts::Fonts, ImageSlider}, GlobalState, @@ -13,6 +12,7 @@ use conrod_core::{ widget::{self, Button, Rectangle, Scrollbar, Text}, widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; widget_ids! { struct Ids { diff --git a/voxygen/src/hud/settings_window/video.rs b/voxygen/src/hud/settings_window/video.rs index c638602f30..e577ecb6d6 100644 --- a/voxygen/src/hud/settings_window/video.rs +++ b/voxygen/src/hud/settings_window/video.rs @@ -5,7 +5,6 @@ use crate::{ img_ids::Imgs, CRITICAL_HP_COLOR, HP_COLOR, LOW_HP_COLOR, MENU_BG, STAMINA_COLOR, TEXT_COLOR, }, - i18n::Localization, render::{ AaMode, CloudMode, FluidMode, LightingMode, PresentMode, RenderMode, ShadowMapMode, ShadowMode, UpscaleMode, @@ -23,6 +22,7 @@ use conrod_core::{ widget_ids, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; use core::convert::TryFrom; +use i18n::Localization; use itertools::Itertools; use std::iter::once; diff --git a/voxygen/src/hud/skillbar.rs b/voxygen/src/hud/skillbar.rs index debac354c9..07eae60057 100644 --- a/voxygen/src/hud/skillbar.rs +++ b/voxygen/src/hud/skillbar.rs @@ -8,7 +8,6 @@ use super::{ use crate::{ game_input::GameInput, hud::{ComboFloater, Position, PositionSpecifier}, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -17,6 +16,7 @@ use crate::{ }, GlobalState, }; +use i18n::Localization; use client::{self, Client}; use common::comp::{ diff --git a/voxygen/src/hud/social.rs b/voxygen/src/hud/social.rs index 8c81cf06a7..6af6a1494e 100644 --- a/voxygen/src/hud/social.rs +++ b/voxygen/src/hud/social.rs @@ -2,11 +2,7 @@ use super::{ img_ids::{Imgs, ImgsRot}, Show, TEXT_COLOR, TEXT_COLOR_3, UI_HIGHLIGHT_0, UI_MAIN, }; - -use crate::{ - i18n::Localization, - ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}, -}; +use crate::ui::{fonts::Fonts, ImageFrame, Tooltip, TooltipManager, Tooltipable}; use client::{self, Client}; use common::{comp::group, uid::Uid}; use conrod_core::{ @@ -14,6 +10,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, Scrollbar, Text, TextEdit}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, Widget, WidgetCommon, }; +use i18n::Localization; use itertools::Itertools; use std::time::Instant; diff --git a/voxygen/src/hud/trade.rs b/voxygen/src/hud/trade.rs index 1aa46459e9..83023f4e7d 100644 --- a/voxygen/src/hud/trade.rs +++ b/voxygen/src/hud/trade.rs @@ -6,7 +6,6 @@ use super::{ }; use crate::{ hud::bag::{BackgroundIds, InventoryScroller}, - i18n::Localization, ui::{ fonts::Fonts, slot::{ContentSize, SlotMaker}, @@ -28,6 +27,7 @@ use conrod_core::{ widget::{self, Button, Image, Rectangle, State as ConrodState, Text}, widget_ids, Color, Colorable, Labelable, Positionable, Sizeable, UiCell, Widget, WidgetCommon, }; +use i18n::Localization; use specs::Entity as EcsEntity; use vek::*; diff --git a/voxygen/src/hud/util.rs b/voxygen/src/hud/util.rs index e383027404..0b7342abf1 100644 --- a/voxygen/src/hud/util.rs +++ b/voxygen/src/hud/util.rs @@ -11,10 +11,9 @@ use common::{ effect::Effect, trade::{Good, SitePrices}, }; +use i18n::Localization; use std::{borrow::Cow, fmt::Write}; -use crate::i18n::Localization; - pub fn price_desc( prices: &Option, item_definition_id: &str, diff --git a/voxygen/src/lib.rs b/voxygen/src/lib.rs index 304710c011..cf4d673534 100644 --- a/voxygen/src/lib.rs +++ b/voxygen/src/lib.rs @@ -33,10 +33,6 @@ pub mod settings; pub mod singleplayer; pub mod window; -// Reexports -pub use crate::error::Error; -pub use i18n; - #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; #[cfg(feature = "egui-ui")] diff --git a/voxygen/src/main.rs b/voxygen/src/main.rs index 2a52b98e68..f7dff8f1c7 100644 --- a/voxygen/src/main.rs +++ b/voxygen/src/main.rs @@ -2,9 +2,9 @@ #![feature(bool_to_option)] #![recursion_limit = "2048"] +use i18n::{self, LocalizationHandle}; use veloren_voxygen::{ audio::AudioFrontend, - i18n::{self, LocalizationHandle}, profile::Profile, run, scene::terrain::SpriteRenderContext, diff --git a/voxygen/src/menu/char_selection/ui/mod.rs b/voxygen/src/menu/char_selection/ui/mod.rs index d48bfafe6e..b26d7c8fbe 100644 --- a/voxygen/src/menu/char_selection/ui/mod.rs +++ b/voxygen/src/menu/char_selection/ui/mod.rs @@ -1,5 +1,4 @@ use crate::{ - i18n::{Localization, LocalizationHandle}, render::UiDrawer, ui::{ self, @@ -26,6 +25,7 @@ use common::{ comp::{self, humanoid, inventory::slot::EquipSlot, Inventory, Item}, LoadoutBuilder, }; +use i18n::{Localization, LocalizationHandle}; //ImageFrame, Tooltip, use crate::settings::Settings; //use std::time::Duration; diff --git a/voxygen/src/menu/main/mod.rs b/voxygen/src/menu/main/mod.rs index 06457e5069..910174901a 100644 --- a/voxygen/src/menu/main/mod.rs +++ b/voxygen/src/menu/main/mod.rs @@ -6,7 +6,6 @@ use super::char_selection::CharSelectionState; #[cfg(feature = "singleplayer")] use crate::singleplayer::Singleplayer; use crate::{ - i18n::LocalizationHandle, render::{Drawer, GlobalsBindGroup}, settings::Settings, window::Event, @@ -20,6 +19,7 @@ use client::{ use client_init::{ClientInit, Error as InitError, Msg as InitMsg}; use common::comp; use common_base::span; +use i18n::LocalizationHandle; use scene::Scene; use std::sync::Arc; use tokio::runtime; diff --git a/voxygen/src/menu/main/ui/connecting.rs b/voxygen/src/menu/main/ui/connecting.rs index 42c090a1c6..49651fdb21 100644 --- a/voxygen/src/menu/main/ui/connecting.rs +++ b/voxygen/src/menu/main/ui/connecting.rs @@ -1,13 +1,12 @@ use super::{ConnectionState, Imgs, Message}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{component::neat_button, style, widget::Image, Element, IcedUi as Ui, Id}, - Graphic, - }, + +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{component::neat_button, style, widget::Image, Element, IcedUi as Ui, Id}, + Graphic, }; use common::assets::{self, AssetExt}; +use i18n::Localization; use iced::{button, Align, Column, Container, Length, Row, Space, Text}; use serde::{Deserialize, Serialize}; diff --git a/voxygen/src/menu/main/ui/disclaimer.rs b/voxygen/src/menu/main/ui/disclaimer.rs index b7225a0edf..885b29bc15 100644 --- a/voxygen/src/menu/main/ui/disclaimer.rs +++ b/voxygen/src/menu/main/ui/disclaimer.rs @@ -1,6 +1,6 @@ use super::Message; +use i18n::{Localization}; use crate::{ - i18n::Localization, ui::{ fonts::IcedFonts as Fonts, ice::{component::neat_button, style, Element}, diff --git a/voxygen/src/menu/main/ui/login.rs b/voxygen/src/menu/main/ui/login.rs index cb8303b951..9234e713d6 100644 --- a/voxygen/src/menu/main/ui/login.rs +++ b/voxygen/src/menu/main/ui/login.rs @@ -1,19 +1,17 @@ use super::{Imgs, LoginInfo, Message, FILL_FRAC_ONE, FILL_FRAC_TWO}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{ - component::neat_button, - style, - widget::{ - compound_graphic::{CompoundGraphic, Graphic}, - BackgroundContainer, Image, Padding, - }, - Element, +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{ + component::neat_button, + style, + widget::{ + compound_graphic::{CompoundGraphic, Graphic}, + BackgroundContainer, Image, Padding, }, + Element, }, }; +use i18n::{LanguageMetadata, Localization}; use iced::{ button, scrollable, text_input, Align, Button, Column, Container, Length, Row, Scrollable, Space, Text, TextInput, @@ -61,7 +59,7 @@ impl Screen { i18n: &Localization, is_selecting_language: bool, selected_language_index: Option, - language_metadatas: &[crate::i18n::LanguageMetadata], + language_metadatas: &[LanguageMetadata], button_style: style::button::Style, version: &str, ) -> Element { @@ -223,7 +221,7 @@ impl LanguageSelectBanner { fonts: &Fonts, imgs: &Imgs, i18n: &Localization, - language_metadatas: &[crate::i18n::LanguageMetadata], + language_metadatas: &[LanguageMetadata], selected_language_index: Option, button_style: style::button::Style, ) -> Element { diff --git a/voxygen/src/menu/main/ui/mod.rs b/voxygen/src/menu/main/ui/mod.rs index 2cb8772994..a013750891 100644 --- a/voxygen/src/menu/main/ui/mod.rs +++ b/voxygen/src/menu/main/ui/mod.rs @@ -5,7 +5,6 @@ mod login; mod servers; use crate::{ - i18n::{LanguageMetadata, LocalizationHandle}, render::UiDrawer, ui::{ self, @@ -16,6 +15,7 @@ use crate::{ }, window, GlobalState, }; +use i18n::{LanguageMetadata, LocalizationHandle}; use iced::{text_input, Column, Container, HorizontalAlignment, Length, Row, Space}; //ImageFrame, Tooltip, use crate::settings::Settings; @@ -193,7 +193,7 @@ impl Controls { .iter() .position(|f| f == &login_info.server); - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); let selected_language_index = language_metadatas .iter() .position(|f| f.language_identifier == settings.language.selected_language); @@ -256,7 +256,7 @@ impl Controls { self.imgs.bg }; - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); // TODO: make any large text blocks scrollable so that if the area is to // small they can still be read @@ -315,7 +315,7 @@ impl Controls { ui: &mut Ui, ) { let servers = &settings.networking.servers; - let mut language_metadatas = crate::i18n::list_localizations(); + let mut language_metadatas = i18n::list_localizations(); match message { Message::Quit => events.push(Event::Quit), @@ -509,7 +509,7 @@ impl MainMenuUi { self.ui.clear_fonts(font); self.controls.fonts = Fonts::load(i18n.fonts(), &mut self.ui).expect("Impossible to load fonts!"); - let language_metadatas = crate::i18n::list_localizations(); + let language_metadatas = i18n::list_localizations(); self.controls.selected_language_index = language_metadatas .iter() .position(|f| f.language_identifier == settings.language.selected_language); diff --git a/voxygen/src/menu/main/ui/servers.rs b/voxygen/src/menu/main/ui/servers.rs index b07804ee63..4e4f20343f 100644 --- a/voxygen/src/menu/main/ui/servers.rs +++ b/voxygen/src/menu/main/ui/servers.rs @@ -1,11 +1,9 @@ use super::{Imgs, Message, FILL_FRAC_ONE}; -use crate::{ - i18n::Localization, - ui::{ - fonts::IcedFonts as Fonts, - ice::{component::neat_button, style, Element}, - }, +use crate::ui::{ + fonts::IcedFonts as Fonts, + ice::{component::neat_button, style, Element}, }; +use i18n::Localization; use iced::{ button, scrollable, Align, Button, Column, Container, Length, Row, Scrollable, Space, Text, }; diff --git a/voxygen/src/session/mod.rs b/voxygen/src/session/mod.rs index f64aed6221..3e7bda0b77 100644 --- a/voxygen/src/session/mod.rs +++ b/voxygen/src/session/mod.rs @@ -35,6 +35,7 @@ use common_net::{ use crate::{ audio::sfx::SfxEvent, + error::Error, game_input::GameInput, hud::{DebugInfo, Event as HudEvent, Hud, HudInfo, LootMessage, PromptDialogSettings}, key_state::KeyState, @@ -43,7 +44,7 @@ use crate::{ scene::{camera, terrain::Interaction, CameraMode, DebugShapeId, Scene, SceneData}, settings::Settings, window::{AnalogGameInput, Event}, - Direction, Error, GlobalState, PlayState, PlayStateResult, + Direction, GlobalState, PlayState, PlayStateResult, }; use hashbrown::HashMap; use settings_change::Language::ChangeLanguage; diff --git a/voxygen/src/session/settings_change.rs b/voxygen/src/session/settings_change.rs index 6f8239c141..62e18e8547 100644 --- a/voxygen/src/session/settings_change.rs +++ b/voxygen/src/session/settings_change.rs @@ -6,7 +6,6 @@ use crate::{ BarNumbers, BuffPosition, ChatTab, CrosshairType, Intro, PressBehavior, ScaleChange, ShortcutNumbers, XpBar, }, - i18n::{LanguageMetadata, LocalizationHandle}, render::RenderMode, settings::{ AudioSettings, ChatSettings, ControlSettings, Fps, GamepadSettings, GameplaySettings, @@ -15,6 +14,7 @@ use crate::{ window::FullScreenSettings, GlobalState, }; +use i18n::{LanguageMetadata, LocalizationHandle}; use vek::*; #[derive(Clone)] diff --git a/voxygen/src/settings/language.rs b/voxygen/src/settings/language.rs index b8e3f8b20e..28fd1840e3 100644 --- a/voxygen/src/settings/language.rs +++ b/voxygen/src/settings/language.rs @@ -1,4 +1,3 @@ -use crate::i18n; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/voxygen/src/ui/cache.rs b/voxygen/src/ui/cache.rs index dcf5041b8e..fd807d48e6 100644 --- a/voxygen/src/ui/cache.rs +++ b/voxygen/src/ui/cache.rs @@ -1,7 +1,7 @@ use super::graphic::{Graphic, GraphicCache, Id as GraphicId}; use crate::{ + error::Error, render::{Mesh, Renderer, Texture, UiTextureBindGroup, UiVertex}, - Error, }; use conrod_core::{text::GlyphCache, widget::Id}; use hashbrown::HashMap; diff --git a/voxygen/src/ui/fonts.rs b/voxygen/src/ui/fonts.rs index 1bd9ac45a1..fe3716542d 100644 --- a/voxygen/src/ui/fonts.rs +++ b/voxygen/src/ui/fonts.rs @@ -1,4 +1,4 @@ -use crate::{i18n, ui::ice::RawFont}; +use crate::ui::ice::RawFont; use common::assets::{self, AssetExt}; pub struct Font { diff --git a/voxygen/src/ui/ice/cache.rs b/voxygen/src/ui/ice/cache.rs index aeb629bc88..e7631e10a4 100644 --- a/voxygen/src/ui/ice/cache.rs +++ b/voxygen/src/ui/ice/cache.rs @@ -1,7 +1,7 @@ use super::graphic::{Graphic, GraphicCache, Id as GraphicId}; use crate::{ + error::Error, render::{Renderer, Texture, UiTextureBindGroup}, - Error, }; use common::assets::{self, AssetExt}; use glyph_brush::GlyphBrushBuilder; diff --git a/voxygen/src/ui/ice/mod.rs b/voxygen/src/ui/ice/mod.rs index 91393ad0cf..d28ef981d9 100644 --- a/voxygen/src/ui/ice/mod.rs +++ b/voxygen/src/ui/ice/mod.rs @@ -15,9 +15,9 @@ use super::{ scale::{Scale, ScaleMode}, }; use crate::{ + error::Error, render::{Renderer, UiDrawer}, window::Window, - Error, }; use common::slowjob::SlowJobPool; use common_base::span; diff --git a/voxygen/src/ui/ice/renderer/mod.rs b/voxygen/src/ui/ice/renderer/mod.rs index 51f4fbf494..2c6ff6014c 100644 --- a/voxygen/src/ui/ice/renderer/mod.rs +++ b/voxygen/src/ui/ice/renderer/mod.rs @@ -14,11 +14,11 @@ use super::{ Font, FontId, RawFont, Rotation, }; use crate::{ + error::Error, render::{ create_ui_quad, create_ui_quad_vert_gradient, DynamicModel, Mesh, Renderer, UiBoundLocals, UiDrawer, UiLocals, UiMode, UiVertex, }, - Error, }; use common::{slowjob::SlowJobPool, util::srgba_to_linear}; use common_base::span; diff --git a/voxygen/src/ui/mod.rs b/voxygen/src/ui/mod.rs index 574dd6ed8c..65b57fb139 100644 --- a/voxygen/src/ui/mod.rs +++ b/voxygen/src/ui/mod.rs @@ -28,12 +28,12 @@ pub use widgets::{ }; use crate::{ + error::Error, render::{ create_ui_quad, create_ui_tri, DynamicModel, Mesh, RenderError, Renderer, UiBoundLocals, UiDrawer, UiLocals, UiMode, UiVertex, }, window::Window, - Error, }; #[rustfmt::skip] use ::image::GenericImageView; diff --git a/voxygen/src/ui/widgets/item_tooltip.rs b/voxygen/src/ui/widgets/item_tooltip.rs index 4dcd6f7cf0..86c3ef1ad4 100644 --- a/voxygen/src/ui/widgets/item_tooltip.rs +++ b/voxygen/src/ui/widgets/item_tooltip.rs @@ -1,12 +1,9 @@ use super::image_frame::ImageFrame; -use crate::{ - hud::{ - get_quality_col, - img_ids::Imgs, - item_imgs::{animate_by_pulse, ItemImgs, ItemKey}, - util, - }, - i18n::Localization, +use crate::hud::{ + get_quality_col, + img_ids::Imgs, + item_imgs::{animate_by_pulse, ItemImgs, ItemKey}, + util, }; use client::Client; use common::{ @@ -22,6 +19,7 @@ use conrod_core::{ widget, widget_ids, Color, Colorable, FontSize, Positionable, Scalar, Sizeable, Ui, UiCell, Widget, WidgetCommon, WidgetStyle, }; +use i18n::Localization; use lazy_static::lazy_static; use std::time::{Duration, Instant}; diff --git a/voxygen/src/window.rs b/voxygen/src/window.rs index 3663e3443a..4f2c3da9d4 100644 --- a/voxygen/src/window.rs +++ b/voxygen/src/window.rs @@ -1,9 +1,10 @@ use crate::{ controller::*, + error::Error, game_input::GameInput, render::Renderer, settings::{ControlSettings, Settings}, - ui, Error, + ui, }; use common_base::span; use crossbeam_channel as channel;