From ee704e171d6d912f9196410db649deb27ab9aa51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=A4rtens?= Date: Mon, 2 Aug 2021 11:05:05 +0200 Subject: [PATCH] move translation check to seperate ci step note: TAGUUID is used to avoid sqlinjections by choosing a branch name --- .gitlab/CI/build.gitlab-ci.yml | 21 +++++++++++++++++- .gitlab/scripts/translation.sh | 4 ++++ .gitlab/scripts/unittest.sh | 1 - voxygen/i18n/src/analysis.rs | 13 ++++++----- voxygen/i18n/src/lib.rs | 2 +- voxygen/i18n/src/path.rs | 5 ++++- voxygen/i18n/src/stats.rs | 40 ++++++++++++++++++---------------- 7 files changed, 57 insertions(+), 29 deletions(-) create mode 100755 .gitlab/scripts/translation.sh diff --git a/.gitlab/CI/build.gitlab-ci.yml b/.gitlab/CI/build.gitlab-ci.yml index 9e68e7ee91..005dd3ef95 100644 --- a/.gitlab/CI/build.gitlab-ci.yml +++ b/.gitlab/CI/build.gitlab-ci.yml @@ -11,6 +11,25 @@ unittests: retry: max: 2 +translation: + extends: .release + stage: build + image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG} + script: + - ln -s /dockercache/target target + - cat ./.gitlab/scripts/translation.sh + - source ./.gitlab/scripts/translation.sh + - TAGUUID="Z$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)" || echo "ignore this returncode, dont ask me why, it works" + - echo $TAGUUID # Use TAGUUID to mitigate https://xkcd.com/327/ in the branch name + - echo 'SET veloren.timestamp = "'"$(git show --no-patch --no-notes --pretty='%cd' HEAD)"'";' > upload.sql + - echo "SET veloren.branch = \$${TAGUUID}\$${CI_COMMIT_REF_NAME}\$${TAGUUID}\$;" >> upload.sql + - echo "SET veloren.sha = \$${TAGUUID}\$${CI_COMMIT_SHA}\$${TAGUUID}\$;" >> upload.sql + - echo '\copy translations ("country_code", "file_name", "translation_key", "status", "git_commit") from '"'translation_analysis.csv' csv header" >> upload.sql + - cat upload.sql + - PGPASSWORD="${CIDBPASSWORD}" PGSSLROOTCERT="./.gitlab/ci-db.crt" psql "sslmode=verify-ca host=grafana.veloren.net port=15432 dbname=translations" -U hgseehzjtsrghtjdcqw -f upload.sql; + retry: + max: 2 + benchmarks: extends: .release stage: build @@ -23,7 +42,7 @@ benchmarks: - cat ./.gitlab/scripts/benchmark.sh - source ./.gitlab/scripts/benchmark.sh - TAGUUID="Z$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)" || echo "ignore this returncode, dont ask me why, it works" - - echo $TAGUUID + - echo $TAGUUID # Use TAGUUID to mitigate https://xkcd.com/327/ in the branch name - echo 'SET veloren.timestamp = "'"$(git show --no-patch --no-notes --pretty='%cd' HEAD)"'";' > upload.sql - echo "SET veloren.branch = \$${TAGUUID}\$${CI_COMMIT_REF_NAME}\$${TAGUUID}\$;" >> upload.sql - echo "SET veloren.sha = \$${TAGUUID}\$${CI_COMMIT_SHA}\$${TAGUUID}\$;" >> upload.sql diff --git a/.gitlab/scripts/translation.sh b/.gitlab/scripts/translation.sh new file mode 100755 index 0000000000..d05fbf0b2a --- /dev/null +++ b/.gitlab/scripts/translation.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export VELOREN_ASSETS="$(pwd)/assets" +rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly +time cargo test --package veloren-voxygen-i18n --lib test_all_localizations -- --nocapture --ignored \ No newline at end of file diff --git a/.gitlab/scripts/unittest.sh b/.gitlab/scripts/unittest.sh index 4bb46b4995..5e693fbed2 100755 --- a/.gitlab/scripts/unittest.sh +++ b/.gitlab/scripts/unittest.sh @@ -1,7 +1,6 @@ #!/bin/bash export VELOREN_ASSETS="$(pwd)/assets" rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly -time cargo test --package veloren-voxygen-i18n --lib test_all_localizations -- --nocapture --ignored && time cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib && ( rm -r target/debug/incremental* || echo "all good" ) && # TMP FIX FOR 2021-03-22-nightly time cargo test \ No newline at end of file diff --git a/voxygen/i18n/src/analysis.rs b/voxygen/i18n/src/analysis.rs index 28b055c908..f1e2d465c7 100644 --- a/voxygen/i18n/src/analysis.rs +++ b/voxygen/i18n/src/analysis.rs @@ -5,7 +5,7 @@ use crate::{ path::{BasePath, LangPath}, raw::{self, RawFragment, RawLanguage}, stats::{ - print_csv_file, print_overall_stats, print_translation_stats, LocalizationAnalysis, + print_csv_stats, print_overall_stats, print_translation_stats, LocalizationAnalysis, LocalizationStats, }, REFERENCE_LANG, @@ -167,10 +167,8 @@ fn gather_results( Some(LocalizationState::Unused) => stats.unused_entries += 1, Some(LocalizationState::UpToDate) => stats.uptodate_entries += 1, }; - if entry.state != Some(LocalizationState::UpToDate) { - let state_keys = state_map.data.get_mut(&entry.state).expect("prefiled"); - state_keys.push((file.clone(), key.to_owned(), entry.commit_id)); - } + let state_keys = state_map.data.get_mut(&entry.state).expect("prefiled"); + state_keys.push((file.clone(), key.to_owned(), entry.commit_id)); } } @@ -209,10 +207,13 @@ pub fn test_specific_localizations( analysis.insert(language_identifier.to_owned(), (state_map, stats)); } + let output = path.root_path().join("translation_analysis.csv"); + let mut f = std::fs::File::create(output).expect("couldn't write csv file"); + //printing for (language_identifier, (state_map, stats)) in &analysis { if csv_enabled { - print_csv_file(state_map); + print_csv_stats(state_map, &mut f); } else { print_translation_stats( language_identifier, diff --git a/voxygen/i18n/src/lib.rs b/voxygen/i18n/src/lib.rs index 249d9e1493..22862c28a8 100644 --- a/voxygen/i18n/src/lib.rs +++ b/voxygen/i18n/src/lib.rs @@ -355,6 +355,6 @@ mod tests { fn test_all_localizations() { // Generate paths let root_dir = common_assets::find_root().expect("Failed to discover repository root"); - crate::analysis::test_all_localizations(&BasePath::new(&root_dir), true, false); + crate::analysis::test_all_localizations(&BasePath::new(&root_dir), true, true); } } diff --git a/voxygen/i18n/src/path.rs b/voxygen/i18n/src/path.rs index a8f961eef5..3f25e55adf 100644 --- a/voxygen/i18n/src/path.rs +++ b/voxygen/i18n/src/path.rs @@ -109,6 +109,8 @@ fn recursive_fragments_paths_in_language( subfolder: &Path, result: &mut Vec, ) -> Result<(), std::io::Error> { + let manifest_path = PathBuf::from(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)); + let template_path = PathBuf::from(&format!("{}.{}", "template", LANG_EXTENSION)); let search_dir = lpath.sub_path(subfolder); for fragment_file in search_dir.read_dir()?.flatten() { let file_type = fragment_file.file_type()?; @@ -117,7 +119,8 @@ fn recursive_fragments_paths_in_language( if file_type.is_dir() { recursive_fragments_paths_in_language(lpath, relative_path, result)?; } else if file_type.is_file() - && relative_path != Path::new(&format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION)) + && relative_path != manifest_path + && relative_path != template_path { result.push(relative_path.to_path_buf()); } diff --git a/voxygen/i18n/src/stats.rs b/voxygen/i18n/src/stats.rs index 9ab93fd062..923840d439 100644 --- a/voxygen/i18n/src/stats.rs +++ b/voxygen/i18n/src/stats.rs @@ -31,22 +31,21 @@ impl LocalizationStats { impl LocalizationAnalysis { pub(crate) fn new(language_identifier: &str) -> Self { let mut data = HashMap::new(); - data.insert(Some(LocalizationState::UpToDate), vec![]); - data.insert(Some(LocalizationState::NotFound), vec![]); - data.insert(Some(LocalizationState::Unused), vec![]); - data.insert(Some(LocalizationState::Outdated), vec![]); - data.insert(None, vec![]); + for key in ALL_LOCALIZATION_STATES.iter() { + data.insert(*key, vec![]); + } Self { language_identifier: language_identifier.to_owned(), data, } } - fn show( + fn show( &self, state: Option, ref_language: &RawLanguage, be_verbose: bool, + output: &mut W, ) { let entries = self.data.get(&state).unwrap_or_else(|| { panic!( @@ -57,7 +56,7 @@ impl LocalizationAnalysis { if entries.is_empty() { return; } - println!("\n\t[{}]", LocalizationState::print(&state)); + writeln!(output, "\n\t[{}]", LocalizationState::print(&state)).unwrap(); for (path, key, commit_id) in entries { if be_verbose { let our_commit = LocalizationAnalysis::print_commit(commit_id); @@ -68,28 +67,30 @@ impl LocalizationAnalysis { .and_then(|s| s.commit_id) .map(|s| format!("{}", s)) .unwrap_or_else(|| "None".to_owned()); - println!("{:60}| {:40} | {:40}", key, our_commit, ref_commit,); + writeln!(output, "{:60}| {:40} | {:40}", key, our_commit, ref_commit).unwrap(); } else { - println!("{}", key); + writeln!(output, "{}", key).unwrap(); } } } - fn csv(&self, state: Option) { + fn csv(&self, state: Option, output: &mut W) { let entries = self .data .get(&state) .unwrap_or_else(|| panic!("called on invalid state: {:?}", state)); for (path, key, commit_id) in entries { let our_commit = LocalizationAnalysis::print_commit(commit_id); - println!( + writeln!( + output, "{},{:?},{},{},{}", self.language_identifier, path, key, LocalizationState::print(&state), our_commit - ); + ) + .unwrap(); } } @@ -126,7 +127,7 @@ pub(crate) fn print_translation_stats( if state == &Some(LocalizationState::UpToDate) { continue; } - state_map.show(*state, ref_language, be_verbose); + state_map.show(*state, ref_language, be_verbose, &mut std::io::stdout()); } println!( @@ -144,14 +145,15 @@ pub(crate) fn print_translation_stats( ); } -pub(crate) fn print_csv_file(state_map: &LocalizationAnalysis) { - println!("country_code,file_name,translation_code,status,git_commit"); +pub(crate) fn print_csv_stats(state_map: &LocalizationAnalysis, output: &mut W) { + writeln!( + output, + "country_code,file_name,translation_key,status,git_commit" + ) + .unwrap(); for state in &ALL_LOCALIZATION_STATES { - if state == &Some(LocalizationState::UpToDate) { - continue; - } - state_map.csv(*state); + state_map.csv(*state, output); } }