From 9a0ac8dccea49c9e521a22a3a1300b98c52f2984 Mon Sep 17 00:00:00 2001 From: juliancoffee Date: Thu, 25 Aug 2022 13:18:06 +0300 Subject: [PATCH 1/3] Make `test_all_localizations` run at unittest --- .gitlab/scripts/unittest.sh | 10 +++++++--- voxygen/i18n/src/analysis.rs | 1 + voxygen/i18n/src/lib.rs | 5 ----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab/scripts/unittest.sh b/.gitlab/scripts/unittest.sh index 06cd7a1b2e..a48387af9f 100755 --- a/.gitlab/scripts/unittest.sh +++ b/.gitlab/scripts/unittest.sh @@ -1,4 +1,8 @@ #!/bin/bash -export VELOREN_ASSETS="$(pwd)/assets" -time cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib && -time cargo test +VELOREN_ASSETS="$(pwd)/assets" +export VELOREN_ASSETS + +time cargo test \ + --package veloren-common-assets asset_tweak::tests \ + --features asset_tweak --lib && +time cargo test --features stat diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index 396bc1b292..48a4351091 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -30,6 +30,7 @@ fn keys_from_file(filepath: &Path) -> Vec { ResourceErr::parsing_error(errs, file.clone(), &content) ) }); + let mut keys = Vec::new(); for entry in ast.body { match entry { diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index ad40e9b8fa..fb23a63198 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -163,10 +163,6 @@ impl assets::Compound for Language { // Here go dragons for id in cache.load_dir::(path, true)?.ids() { - if id.ends_with("_manifest") { - continue; - } - match cache.load(id) { Ok(handle) => { let source: &raw::Resource = &*handle.read(); @@ -518,7 +514,6 @@ mod tests { } #[test] - #[ignore] #[cfg(feature = "stat")] // Generate translation stats fn test_all_localizations() { From f2eb1fe7c8e0c564ddccca3346d09f164382e30d Mon Sep 17 00:00:00 2001 From: juliancoffee Date: Thu, 25 Aug 2022 14:13:28 +0300 Subject: [PATCH 2/3] Make i18n csv own binary * Put file that needs care back into csv --- .gitlab/scripts/translation.sh | 5 +-- voxygen/i18n/Cargo.toml | 4 +++ voxygen/i18n/src/analysis.rs | 9 ++--- voxygen/i18n/src/bin/i18n-csv.rs | 57 ++++++++++++++++++++++++++++++++ voxygen/i18n/src/lib.rs | 44 +++--------------------- 5 files changed, 69 insertions(+), 50 deletions(-) create mode 100644 voxygen/i18n/src/bin/i18n-csv.rs diff --git a/.gitlab/scripts/translation.sh b/.gitlab/scripts/translation.sh index 4410ae57bb..4f21367d69 100755 --- a/.gitlab/scripts/translation.sh +++ b/.gitlab/scripts/translation.sh @@ -2,7 +2,4 @@ VELOREN_ASSETS="$(pwd)/assets" export VELOREN_ASSETS -time cargo test --package veloren-voxygen-i18n \ - --lib test_all_localizations \ - --features="stat" \ - -- --nocapture --ignored +time cargo run --bin i18n-csv --features="stat" diff --git a/voxygen/i18n/Cargo.toml b/voxygen/i18n/Cargo.toml index c5ff1c56d7..7f69709878 100644 --- a/voxygen/i18n/Cargo.toml +++ b/voxygen/i18n/Cargo.toml @@ -27,6 +27,10 @@ fluent-syntax = { git = "https://github.com/juliancoffee/fluent-rs.git", branch name = "i18n-check" required-features = ["bin"] +[[bin]] +name = "i18n-csv" +required-features = ["stat"] + [features] bin = ["clap"] stat = [] diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index 48a4351091..db5dafdb39 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -37,7 +37,7 @@ fn keys_from_file(filepath: &Path) -> Vec { Entry::Message(m) => { keys.push(MsgId { key: m.id.name.to_owned(), - file: Some(file.clone()), + file: file.clone(), }); }, Entry::Term(_) @@ -80,7 +80,7 @@ fn keys(from: &Path, tree: &Walk) -> Vec { #[derive(Clone, Debug)] pub struct MsgId { pub key: String, - pub file: Option, + pub file: String, } // TODO: @@ -122,10 +122,7 @@ impl ReferenceLanguage { if let Some(key) = keys.iter().find(|MsgId { key, .. }| &ref_key.key == key) { stats.up_to_date.push(key.clone()); } else { - stats.not_found.push(MsgId { - key: ref_key.key.clone(), - file: None, - }); + stats.not_found.push(ref_key.clone()); } } diff --git a/voxygen/i18n/src/bin/i18n-csv.rs b/voxygen/i18n/src/bin/i18n-csv.rs new file mode 100644 index 0000000000..d26706f2ed --- /dev/null +++ b/voxygen/i18n/src/bin/i18n-csv.rs @@ -0,0 +1,57 @@ +use common_assets::find_root; +use std::{fs, io::Write, path::Path}; +use veloren_voxygen_i18n::{ + analysis::{Language, ReferenceLanguage}, + list_localizations, REFERENCE_LANG, +}; + +fn main() { + let root = find_root().unwrap(); + let output = root.join("translation_analysis.csv"); + let mut f = fs::File::create(output).expect("couldn't write csv file"); + + writeln!( + f, + "country_code,file_name,translation_key,status,git_commit" + ) + .unwrap(); + + let i18n_directory = root.join("assets/voxygen/i18n"); + let reference = ReferenceLanguage::at(&i18n_directory.join(REFERENCE_LANG)); + + let list = list_localizations(); + let file = |filename| { + let file = Path::new(&filename) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(""); + + format!("{file}") + }; + for meta in list { + let code = meta.language_identifier; + let lang = Language { + code: code.clone(), + path: i18n_directory.join(code.clone()), + }; + let stats = reference.compare_with(&lang); + for key in stats.up_to_date { + let code = &code; + let filename = file(key.file); + let key = &key.key; + writeln!(f, "{code},{filename},{key},UpToDate,None").unwrap(); + } + for key in stats.not_found { + let code = &code; + let filename = file(key.file); + let key = &key.key; + writeln!(f, "{code},{filename},{key},NotFound,None").unwrap(); + } + for key in stats.unused { + let code = &code; + let filename = file(key.file); + let key = &key.key; + writeln!(f, "{code},{filename},{key},Unused,None").unwrap(); + } + } +} diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index fb23a63198..6556dc980c 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -515,61 +515,25 @@ mod tests { #[test] #[cfg(feature = "stat")] - // Generate translation stats - fn test_all_localizations() { + fn test_strict_all_localizations() { use analysis::{Language, ReferenceLanguage}; use assets::find_root; - use std::{fs, io::Write, path::Path}; let root = find_root().unwrap(); - let output = root.join("translation_analysis.csv"); - let mut f = fs::File::create(output).expect("couldn't write csv file"); - - writeln!( - f, - "country_code,file_name,translation_key,status,git_commit" - ) - .unwrap(); - let i18n_directory = root.join("assets/voxygen/i18n"); let reference = ReferenceLanguage::at(&i18n_directory.join(REFERENCE_LANG)); let list = list_localizations(); - let file = |filename: Option| { - let file = filename - .as_ref() - .map(|s| Path::new(s)) - .and_then(|p| p.file_name()) - .and_then(|s| s.to_str()) - .unwrap_or("None"); - format!("{file}") - }; for meta in list { let code = meta.language_identifier; let lang = Language { code: code.clone(), path: i18n_directory.join(code.clone()), }; - let stats = reference.compare_with(&lang); - for key in stats.up_to_date { - let code = &code; - let filename = &file(key.file); - let key = &key.key; - writeln!(f, "{code},{filename},{key},UpToDate,None").unwrap(); - } - for key in stats.not_found { - let code = &code; - let filename = &file(key.file); - let key = &key.key; - writeln!(f, "{code},{filename},{key},NotFound,None").unwrap(); - } - for key in stats.unused { - let code = &code; - let filename = &file(key.file); - let key = &key.key; - writeln!(f, "{code},{filename},{key},Unused,None").unwrap(); - } + // TODO: somewhere here should go check that all needed + // versions are given + reference.compare_with(&lang); } } } From ae399b3c7c22f7eb68024638a3b7297bf5de881d Mon Sep 17 00:00:00 2001 From: juliancoffee Date: Thu, 25 Aug 2022 14:20:54 +0300 Subject: [PATCH 3/3] Remove cfg(feature = stat) for localization test --- .gitlab/scripts/unittest.sh | 2 +- voxygen/i18n/src/lib.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/unittest.sh b/.gitlab/scripts/unittest.sh index a48387af9f..1b346c6bdd 100755 --- a/.gitlab/scripts/unittest.sh +++ b/.gitlab/scripts/unittest.sh @@ -5,4 +5,4 @@ export VELOREN_ASSETS time cargo test \ --package veloren-common-assets asset_tweak::tests \ --features asset_tweak --lib && -time cargo test --features stat +time cargo test diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 6556dc980c..435bb71884 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -3,7 +3,7 @@ mod raw; use error::ResourceErr; -#[cfg(any(feature = "bin", feature = "stat"))] +#[cfg(any(feature = "bin", feature = "stat", test))] pub mod analysis; use fluent_bundle::{bundle::FluentBundle, FluentResource}; @@ -514,7 +514,6 @@ mod tests { } #[test] - #[cfg(feature = "stat")] fn test_strict_all_localizations() { use analysis::{Language, ReferenceLanguage}; use assets::find_root;