Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

base framework to print csv

Browse Source
This commit is contained in:
Dr. Dystopia 2021-07-23 13:32:00 +02:00 committed by Marcel Märtens
parent c9c32eea65
commit c501b2eb70
6 changed files with 570 additions and 495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
voxygen/i18n/src/analysis.rs
View File

@ -1,7 +1,7 @@
use ron::de::from_bytes;
use std::path::{Path, PathBuf};
use crate::data::{
use crate::raw::{
i18n_directories, LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG,
};
use hashbrown::{HashMap, HashSet};
@ -27,6 +27,7 @@ struct LocalizationStats {
#[derive(Default)]
struct LocalizationAnalysis {
uptodate: Vec<(String, Option<git2::Oid>)>,
notfound: Vec<(String, Option<git2::Oid>)>,
unused: Vec<(String, Option<git2::Oid>)>,
outdated: Vec<(String, Option<git2::Oid>)>,
@ -39,11 +40,11 @@ impl LocalizationAnalysis {
state: LocalizationState,
) -> Option<&mut Vec<(String, Option<git2::Oid>)>> {
match state {
LocalizationState::UpToDate => Some(&mut self.uptodate),
LocalizationState::NotFound => Some(&mut self.notfound),
LocalizationState::Unused => Some(&mut self.unused),
LocalizationState::Outdated => Some(&mut self.outdated),
LocalizationState::Unknown => Some(&mut self.unknown),
_ => None,
}
}
@ -53,9 +54,7 @@ impl LocalizationAnalysis {
be_verbose: bool,
ref_i18n_map: &HashMap<String, LocalizationEntryState>,
) {
let entries = self
.get_mut(state)
.unwrap_or_else(|| panic!("called on invalid state: {:?}", state));
let entries = self.unwrap_entries(state);
if entries.is_empty() {
return;
}
@ -63,9 +62,7 @@ impl LocalizationAnalysis {
entries.sort();
for (key, commit_id) in entries {
if be_verbose {
let our_commit = commit_id
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_owned());
let our_commit = LocalizationAnalysis::create_our_commit(commit_id);
let ref_commit = ref_i18n_map
.get(key)
.and_then(|s| s.commit_id)
@ -77,6 +74,32 @@ impl LocalizationAnalysis {
}
}
}
//TODO: Add which file each faulty translation is in
fn csv(&mut self, state: LocalizationState) {
let entries = self.unwrap_entries(state);
for (key, commit_id) in entries {
let our_commit = LocalizationAnalysis::create_our_commit(commit_id);
println!(
"{},{},{},{:?},{}",
"sv", "_manifest.yml", key, state, our_commit
);
}
}
fn unwrap_entries(
&mut self,
state: LocalizationState,
) -> &mut Vec<(String, Option<git2::Oid>)> {
self.get_mut(state)
.unwrap_or_else(|| panic!("called on invalid state: {:?}", state))
}
fn create_our_commit(commit_id: &mut Option<git2::Oid>) -> String {
commit_id
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_owned())
}
}
#[derive(Copy, Clone, Debug)]
@ -329,14 +352,7 @@ fn test_localization_directory(
let mut state_map = LocalizationAnalysis::default();
let result = gather_results(current_i18n, &mut state_map);
print_translation_stats(
i18n_references,
&result,
&mut state_map,
be_verbose,
relfile,
ref_manifest,
);
print_csv_file(&mut state_map, relfile);
Some(result)
}
@ -490,6 +506,16 @@ fn print_translation_stats(
);
}
fn print_csv_file(state_map: &mut LocalizationAnalysis, relfile: PathBuf) {
println!("country_code,file_name,translation_code,status,git_commit");
state_map.csv(LocalizationState::UpToDate);
state_map.csv(LocalizationState::NotFound);
state_map.csv(LocalizationState::Unused);
state_map.csv(LocalizationState::Outdated);
state_map.csv(LocalizationState::Unknown);
}
/// Test one language
/// `code` - name of the directory in assets (de_DE for example)
/// `root_dir` - absolute path to main repo
@ -548,7 +574,12 @@ pub fn test_specific_localization(
/// `assets_path` - relative path to asset directory (right now it is
/// 'assets/voxygen/i18n')
/// csv_enabled - generate csv files in target folder
pub fn test_all_localizations(root_dir: &Path, assets_path: &Path, be_verbose: bool, csv_enabled: bool) {
pub fn test_all_localizations(
root_dir: &Path,
assets_path: &Path,
be_verbose: bool,
csv_enabled: bool,
) {
let ref_lang_dir = assets_path.join(REFERENCE_LANG);
let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron");

7
voxygen/i18n/src/bin/i18n-check.rs
View File

@ -50,7 +50,12 @@ fn main() {
);
}
if matches.is_present("test") {
analysis::test_all_localizations(&root, &asset_path, matches.is_present("verbose"), csv_enabled);
analysis::test_all_localizations(
&root,
&asset_path,
matches.is_present("verbose"),
csv_enabled,
);
}
if matches.is_present("verify") {
verification::verify_all_localizations(&root, &asset_path);

437
voxygen/i18n/src/data.rs
View File

@ -1,437 +0,0 @@
use crate::assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle};
use deunicode::deunicode;
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use std::{
fs, io,
path::{Path, PathBuf},
};
use tracing::warn;
/// The reference language, aka the more up-to-date localization data.
/// Also the default language at first startup.
pub const REFERENCE_LANG: &str = "en";
pub const LANG_MANIFEST_FILE: &str = "_manifest";
/// How a language can be described
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LanguageMetadata {
/// A human friendly language name (e.g. "English (US)")
pub language_name: String,
/// A short text identifier for this language (e.g. "en_US")
///
/// On the opposite of `language_name` that can change freely,
/// `language_identifier` value shall be stable in time as it
/// is used by setting components to store the language
/// selected by the user.
pub language_identifier: String,
}
/// Store font metadata
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Font {
/// Key to retrieve the font in the asset system
pub asset_key: String,
/// Scale ratio to resize the UI text dynamicly
scale_ratio: f32,
}
impl Font {
/// Scale input size to final UI size
pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
}
/// Store font metadata
pub type Fonts = HashMap<String, Font>;
/// Raw localization data, expect the strings to not be loaded here
/// However, metadata informations are correct
/// See `Language` for more info on each attributes
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
pub(crate) struct RawLocalization {
pub(crate) convert_utf8_to_ascii: bool,
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
pub(crate) string_map: HashMap<String, String>,
pub(crate) vector_map: HashMap<String, Vec<String>>,
}
/// Store internationalization data
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Language {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub(crate) string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub(crate) vector_map: HashMap<String, Vec<String>>,
/// Whether to convert the input text encoded in UTF-8
/// into a ASCII version by using the `deunicode` crate.
pub(crate) convert_utf8_to_ascii: bool,
/// Font configuration is stored here
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
}
/// Store internationalization maps
/// These structs are meant to be merged into a Language
#[derive(Debug, PartialEq, Serialize, Deserialize)]
pub(crate) struct LocalizationFragment {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub(crate) string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub(crate) vector_map: HashMap<String, Vec<String>>,
}
impl Language {
/// Get a localized text from the given key
pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> {
self.string_map.get(key).map(String::as_str)
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> {
self.vector_map.get(key).and_then(|v| {
if v.is_empty() {
None
} else {
Some(v[index as usize % v.len()].as_str())
}
})
}
}
impl Default for Language {
fn default() -> Self {
Self {
string_map: HashMap::default(),
vector_map: HashMap::default(),
..Default::default()
}
}
}
impl From<RawLocalization> for Language {
fn from(raw: RawLocalization) -> Self {
Self {
string_map: raw.string_map,
vector_map: raw.vector_map,
convert_utf8_to_ascii: raw.convert_utf8_to_ascii,
fonts: raw.fonts,
metadata: raw.metadata,
}
}
}
impl From<RawLocalization> for LocalizationFragment {
fn from(raw: RawLocalization) -> Self {
Self {
string_map: raw.string_map,
vector_map: raw.vector_map,
}
}
}
impl assets::Asset for RawLocalization {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Asset for LocalizationFragment {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Compound for Language {
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
asset_key: &str,
) -> Result<Self, assets::Error> {
let raw = cache
.load::<RawLocalization>(&[asset_key, ".", LANG_MANIFEST_FILE].concat())?
.cloned();
let mut localization = Language::from(raw);
// Walk through files in the folder, collecting localization fragment to merge
// inside the asked_localization
for localization_asset in cache
.load_dir::<LocalizationFragment>(asset_key, true)?
.iter()
{
localization
.string_map
.extend(localization_asset.read().string_map.clone());
localization
.vector_map
.extend(localization_asset.read().vector_map.clone());
}
// Update the text if UTF-8 to ASCII conversion is enabled
if localization.convert_utf8_to_ascii {
for value in localization.string_map.values_mut() {
*value = deunicode(value);
}
for value in localization.vector_map.values_mut() {
*value = value.iter().map(|s| deunicode(s)).collect();
}
}
localization.metadata.language_name = deunicode(&localization.metadata.language_name);
Ok(localization)
}
}
/// the central data structure to handle localization in veloren
// inherit Copy+Clone from AssetHandle
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct LocalizationHandle {
active: AssetHandle<Language>,
fallback: Option<AssetHandle<Language>>,
pub use_english_fallback: bool,
}
// RAII guard returned from Localization::read(), resembles AssetGuard
pub struct LocalizationGuard {
active: AssetGuard<Language>,
fallback: Option<AssetGuard<Language>>,
}
// arbitrary choice to minimize changing all of veloren
pub type Localization = LocalizationGuard;
impl LocalizationGuard {
/// Get a localized text from the given key
///
/// First lookup is done in the active language, second in
/// the fallback (if present).
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get<'a>(&'a self, key: &'a str) -> &str {
self.active.get(key).unwrap_or_else(|| {
self.fallback
.as_ref()
.and_then(|f| f.get(key))
.unwrap_or(key)
})
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str {
self.active.get_variation(key, index).unwrap_or_else(|| {
self.fallback
.as_ref()
.and_then(|f| f.get_variation(key, index))
.unwrap_or(key)
})
}
/// Return the missing keys compared to the reference language
fn list_missing_entries(&self) -> (HashSet<String>, HashSet<String>) {
if let Some(ref_lang) = &self.fallback {
let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect();
let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect();
let strings = reference_string_keys
.difference(&string_keys)
.cloned()
.collect();
let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect();
let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect();
let vectors = reference_vector_keys
.difference(&vector_keys)
.cloned()
.collect();
(strings, vectors)
} else {
(HashSet::default(), HashSet::default())
}
}
/// Log missing entries (compared to the reference language) as warnings
pub fn log_missing_entries(&self) {
let (missing_strings, missing_vectors) = self.list_missing_entries();
for missing_key in missing_strings {
warn!(
"[{:?}] Missing string key {:?}",
self.metadata().language_identifier,
missing_key
);
}
for missing_key in missing_vectors {
warn!(
"[{:?}] Missing vector key {:?}",
self.metadata().language_identifier,
missing_key
);
}
}
pub fn fonts(&self) -> &Fonts { &self.active.fonts }
pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
}
impl LocalizationHandle {
pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
self.use_english_fallback = use_english_fallback;
}
pub fn read(&self) -> LocalizationGuard {
LocalizationGuard {
active: self.active.read(),
fallback: if self.use_english_fallback {
self.fallback.map(|f| f.read())
} else {
None
},
}
}
pub fn load(specifier: &str) -> Result<Self, crate::assets::Error> {
let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat();
let language_key = ["voxygen.i18n.", specifier].concat();
let is_default = language_key == default_key;
Ok(Self {
active: Language::load(&language_key)?,
fallback: if is_default {
None
} else {
Language::load(&default_key).ok()
},
use_english_fallback: false,
})
}
pub fn load_expect(specifier: &str) -> Self {
Self::load(specifier).expect("Can't load language files")
}
pub fn reloaded(&mut self) -> bool { self.active.reloaded() }
}
struct FindManifests;
impl assets::Compound for FindManifests {
fn load<S: assets::Source>(_: &assets::AssetCache<S>, _: &str) -> Result<Self, assets::Error> {
Ok(Self)
}
}
impl assets::DirLoadable for FindManifests {
fn select_ids<S: assets::Source + ?Sized>(
source: &S,
specifier: &str,
) -> io::Result<Vec<assets::SharedString>> {
let mut specifiers = Vec::new();
source.read_dir(specifier, &mut |entry| {
if let DirEntry::Directory(spec) = entry {
let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat();
if source.exists(DirEntry::File(&manifest_spec, "ron")) {
specifiers.push(manifest_spec.into());
}
}
})?;
Ok(specifiers)
}
}
#[derive(Clone, Debug)]
struct LocalizationList(Vec<LanguageMetadata>);
impl assets::Compound for LocalizationList {
fn load<S: assets::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, assets::Error> {
// List language directories
let languages = assets::load_dir::<FindManifests>(specifier, false)
.unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e))
.ids()
.filter_map(|spec| cache.load::<RawLocalization>(spec).ok())
.map(|localization| localization.read().metadata.clone())
.collect();
Ok(LocalizationList(languages))
}
}
/// Load all the available languages located in the voxygen asset directory
pub fn list_localizations() -> Vec<LanguageMetadata> {
LocalizationList::load_expect_cloned("voxygen.i18n").0
}
/// List localization directories as a `PathBuf` vector
pub fn i18n_directories(i18n_dir: &Path) -> Vec<PathBuf> {
fs::read_dir(i18n_dir)
.unwrap()
.map(|res| res.map(|e| e.path()).unwrap())
.filter(|e| e.is_dir())
.collect()
}
#[cfg(test)]
mod tests {
use super::assets;
// Test that localization list is loaded (not empty)
#[test]
fn test_localization_list() {
let list = super::list_localizations();
assert!(!list.is_empty());
}
// Test that reference language can be loaded
#[test]
fn test_localization_handle() {
let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG);
}
// Test to verify all languages that they are VALID and loadable, without
// need of git just on the local assets folder
#[test]
fn verify_all_localizations() {
// Generate paths
let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/");
let root_dir = assets::find_root().expect("Failed to discover repository root");
crate::verification::verify_all_localizations(&root_dir, i18n_asset_path);
}
// Test to verify all languages and print missing and faulty localisation
#[test]
#[ignore]
fn test_all_localizations() {
// Options
let be_verbose = true;
// Generate paths
let i18n_asset_path = std::path::Path::new("assets/voxygen/i18n/");
let root_dir = assets::find_root().expect("Failed to discover repository root");
crate::analysis::test_all_localizations(&root_dir, i18n_asset_path, be_verbose);
}
}

364
voxygen/i18n/src/lib.rs
View File

@ -1,7 +1,365 @@
#[cfg(any(feature = "bin", test))]
pub mod analysis;
mod data;
pub mod raw;
pub mod verification;
use common_assets as assets;
pub use data::*;
use common_assets::{self, source::DirEntry, AssetExt, AssetGuard, AssetHandle};
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use std::{
fs, io,
path::{Path, PathBuf},
};
use tracing::warn;
use raw::{RawManifest, RawFragment, RawLanguage};
/// The reference language, aka the more up-to-date localization data.
/// Also the default language at first startup.
pub const REFERENCE_LANG: &str = "en";
pub const LANG_MANIFEST_FILE: &str = "_manifest";
pub(crate) const LANG_EXTENSION: &str = "ron";
/// How a language can be described
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LanguageMetadata {
/// A human friendly language name (e.g. "English (US)")
pub language_name: String,
/// A short text identifier for this language (e.g. "en_US")
///
/// On the opposite of `language_name` that can change freely,
/// `language_identifier` value shall be stable in time as it
/// is used by setting components to store the language
/// selected by the user.
pub language_identifier: String,
}
/// Store font metadata
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Font {
/// Key to retrieve the font in the asset system
pub asset_key: String,
/// Scale ratio to resize the UI text dynamically
scale_ratio: f32,
}
impl Font {
/// Scale input size to final UI size
pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
}
/// Store font metadata
pub type Fonts = HashMap<String, Font>;
/// Store internationalization data
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Language {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub(crate) string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub(crate) vector_map: HashMap<String, Vec<String>>,
/// Whether to convert the input text encoded in UTF-8
/// into a ASCII version by using the `deunicode` crate.
pub(crate) convert_utf8_to_ascii: bool,
/// Font configuration is stored here
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
}
impl Language {
/// Get a localized text from the given key
pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> {
self.string_map.get(key).map(String::as_str)
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> {
self.vector_map.get(key).and_then(|v| {
if v.is_empty() {
None
} else {
Some(v[index as usize % v.len()].as_str())
}
})
}
}
impl common_assets::Compound for Language {
fn load<S: common_assets::source::Source>(
cache: &common_assets::AssetCache<S>,
asset_key: &str,
) -> Result<Self, common_assets::Error> {
let manifest = cache
.load::<RawManifest>(&[asset_key, ".", LANG_MANIFEST_FILE].concat())?
.cloned();
// Walk through files in the folder, collecting localization fragment to merge
// inside the asked_localization
let mut fragments = HashMap::new();
for fragment_asset in cache
.load_dir::<RawFragment>(asset_key, true)?
.iter()
{
let read = fragment_asset.read();
fragments.insert(PathBuf::from(fragment_asset.id()), read.clone());
}
Ok(Language::from(RawLanguage{
manifest,
fragments,
}))
}
}
/// the central data structure to handle localization in veloren
// inherit Copy+Clone from AssetHandle
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct LocalizationHandle {
active: AssetHandle<Language>,
fallback: Option<AssetHandle<Language>>,
pub use_english_fallback: bool,
}
// RAII guard returned from Localization::read(), resembles AssetGuard
pub struct LocalizationGuard {
active: AssetGuard<Language>,
fallback: Option<AssetGuard<Language>>,
}
// arbitrary choice to minimize changing all of veloren
pub type Localization = LocalizationGuard;
impl LocalizationGuard {
/// Get a localized text from the given key
///
/// First lookup is done in the active language, second in
/// the fallback (if present).
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get<'a>(&'a self, key: &'a str) -> &str {
self.active.get(key).unwrap_or_else(|| {
self.fallback
.as_ref()
.and_then(|f| f.get(key))
.unwrap_or(key)
})
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str {
self.active.get_variation(key, index).unwrap_or_else(|| {
self.fallback
.as_ref()
.and_then(|f| f.get_variation(key, index))
.unwrap_or(key)
})
}
/// Return the missing keys compared to the reference language
fn list_missing_entries(&self) -> (HashSet<String>, HashSet<String>) {
if let Some(ref_lang) = &self.fallback {
let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect();
let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect();
let strings = reference_string_keys
.difference(&string_keys)
.cloned()
.collect();
let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect();
let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect();
let vectors = reference_vector_keys
.difference(&vector_keys)
.cloned()
.collect();
(strings, vectors)
} else {
(HashSet::default(), HashSet::default())
}
}
/// Log missing entries (compared to the reference language) as warnings
pub fn log_missing_entries(&self) {
let (missing_strings, missing_vectors) = self.list_missing_entries();
for missing_key in missing_strings {
warn!(
"[{:?}] Missing string key {:?}",
self.metadata().language_identifier,
missing_key
);
}
for missing_key in missing_vectors {
warn!(
"[{:?}] Missing vector key {:?}",
self.metadata().language_identifier,
missing_key
);
}
}
pub fn fonts(&self) -> &Fonts { &self.active.fonts }
pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
}
impl LocalizationHandle {
pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
self.use_english_fallback = use_english_fallback;
}
pub fn read(&self) -> LocalizationGuard {
LocalizationGuard {
active: self.active.read(),
fallback: if self.use_english_fallback {
self.fallback.map(|f| f.read())
} else {
None
},
}
}
pub fn load(specifier: &str) -> Result<Self, common_assets::Error> {
let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat();
let language_key = ["voxygen.i18n.", specifier].concat();
let is_default = language_key == default_key;
Ok(Self {
active: Language::load(&language_key)?,
fallback: if is_default {
None
} else {
Language::load(&default_key).ok()
},
use_english_fallback: false,
})
}
pub fn load_expect(specifier: &str) -> Self {
Self::load(specifier).expect("Can't load language files")
}
pub fn reloaded(&mut self) -> bool { self.active.reloaded() }
}
struct FindManifests;
impl common_assets::Compound for FindManifests {
fn load<S: common_assets::Source>(_: &common_assets::AssetCache<S>, _: &str) -> Result<Self, common_assets::Error> {
Ok(Self)
}
}
impl common_assets::DirLoadable for FindManifests {
fn select_ids<S: common_assets::Source + ?Sized>(
source: &S,
specifier: &str,
) -> io::Result<Vec<common_assets::SharedString>> {
let mut specifiers = Vec::new();
source.read_dir(specifier, &mut |entry| {
if let DirEntry::Directory(spec) = entry {
let manifest_spec = [spec, ".", LANG_MANIFEST_FILE].concat();
if source.exists(DirEntry::File(&manifest_spec, LANG_EXTENSION)) {
specifiers.push(manifest_spec.into());
}
}
})?;
Ok(specifiers)
}
}
#[derive(Clone, Debug)]
struct LocalizationList(Vec<LanguageMetadata>);
impl common_assets::Compound for LocalizationList {
fn load<S: common_assets::Source>(
cache: &common_assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, common_assets::Error> {
// List language directories
let languages = common_assets::load_dir::<FindManifests>(specifier, false)
.unwrap_or_else(|e| panic!("Failed to get manifests from {}: {:?}", specifier, e))
.ids()
.filter_map(|spec| cache.load::<RawManifest>(spec).ok())
.map(|localization| localization.read().metadata.clone())
.collect();
Ok(LocalizationList(languages))
}
}
/// Load all the available languages located in the voxygen asset directory
pub fn list_localizations() -> Vec<LanguageMetadata> {
LocalizationList::load_expect_cloned("voxygen.i18n").0
}
/// List localization directories as a `PathBuf` vector
pub fn i18n_directories(i18n_dir: &Path) -> Vec<PathBuf> {
fs::read_dir(i18n_dir)
.unwrap()
.map(|res| res.map(|e| e.path()).unwrap())
.filter(|e| e.is_dir())
.collect()
}
#[cfg(test)]
mod tests {
use std::path::Path;
use common_assets;
// Test that localization list is loaded (not empty)
#[test]
fn test_localization_list() {
let list = super::list_localizations();
assert!(!list.is_empty());
}
// Test that reference language can be loaded
#[test]
fn test_localization_handle() {
let _ = super::LocalizationHandle::load_expect(super::REFERENCE_LANG);
}
// Test to verify all languages that they are VALID and loadable, without
// need of git just on the local assets folder
#[test]
fn verify_all_localizations() {
// Generate paths
let i18n_root_path = Path::new("assets/voxygen/i18n/");
let root_dir = common_assets::find_root().expect("Failed to discover repository root");
crate::verification::verify_all_localizations(&root_dir, i18n_root_path);
}
// Test to verify all languages and print missing and faulty localisation
#[test]
#[ignore]
fn test_all_localizations() {
// Options
let be_verbose = true;
// Generate paths
let i18n_root_path = Path::new("assets/voxygen/i18n/");
let root_dir = common_assets::find_root().expect("Failed to discover repository root");
crate::analysis::test_all_localizations(&root_dir, i18n_root_path, be_verbose);
}
}

136
voxygen/i18n/src/raw.rs Normal file
View File

@ -0,0 +1,136 @@
//! handle the loading of a `Language`
use hashbrown::hash_map::HashMap;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use std::fs;
use ron::de::from_reader;
use deunicode::deunicode;
use crate::{Fonts, LanguageMetadata, LANG_MANIFEST_FILE, LANG_EXTENSION};
use crate::Language;
/// Raw localization metadata from LANG_MANIFEST_FILE file
/// See `Language` for more info on each attributes
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
pub(crate) struct RawManifest {
pub(crate) convert_utf8_to_ascii: bool,
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
}
/// Raw localization data from one specific file
/// These structs are meant to be merged into a Language
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
pub(crate) struct RawFragment {
pub(crate) string_map: HashMap<String, String>,
pub(crate) vector_map: HashMap<String, Vec<String>>,
}
pub(crate) struct RawLanguage {
pub(crate) manifest: RawManifest,
pub(crate) fragments: HashMap<PathBuf, RawFragment>,
}
#[derive(Debug)]
pub(crate) enum RawError {
RonError(ron::Error),
}
/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` folders
pub(crate) fn load_manifest(i18n_root_path: &Path, language_identifier: &str) -> Result<RawManifest, common_assets::Error> {
let manifest_file = i18n_root_path.join(language_identifier).join(format!("{}.{}", LANG_MANIFEST_FILE, LANG_EXTENSION));
println!("file , {:?}", manifest_file);
let f = fs::File::open(&manifest_file)?;
Ok(from_reader(f).map_err(RawError::RonError)?)
}
/// `i18n_root_path` - absolute path to i18n path which contains `en`, `de_DE`, `fr_FR` files
pub(crate) fn load_raw_language(i18n_root_path: &Path, manifest: RawManifest) -> Result<RawLanguage, common_assets::Error> {
// Walk through each file in the directory
let mut fragments = HashMap::new();
let language_identifier = &manifest.metadata.language_identifier;
let language_dir = i18n_root_path.join(language_identifier);
for fragment_file in language_dir.read_dir().unwrap().flatten() {
let file_type = fragment_file.file_type()?;
if file_type.is_dir() {
// TODO: recursive
continue;
}
if file_type.is_file() {
let full_path = fragment_file.path();
let relative_path = full_path.strip_prefix(&i18n_root_path).unwrap();
let f = fs::File::open(&full_path)?;
let fragment = from_reader(f).map_err(RawError::RonError)?;
fragments.insert(relative_path.to_path_buf(), fragment);
}
}
Ok(RawLanguage{
manifest,
fragments,
})
}
impl From<RawLanguage> for Language {
fn from(raw: RawLanguage) -> Self {
let mut string_map = HashMap::new();
let mut vector_map = HashMap::new();
for (_, fragment) in raw.fragments {
string_map.extend(fragment.string_map);
vector_map.extend(fragment.vector_map);
}
let convert_utf8_to_ascii = raw.manifest.convert_utf8_to_ascii;
// Update the text if UTF-8 to ASCII conversion is enabled
if convert_utf8_to_ascii {
for value in string_map.values_mut() {
*value = deunicode(value);
}
for value in vector_map.values_mut() {
*value = value.iter().map(|s| deunicode(s)).collect();
}
}
let mut metadata = raw.manifest.metadata;
metadata.language_name = deunicode(&metadata.language_name);
Self {
string_map,
vector_map,
convert_utf8_to_ascii,
fonts: raw.manifest.fonts,
metadata: metadata,
}
}
}
impl core::fmt::Display for RawError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
RawError::RonError(e) => write!(f, "{}", e),
}
}
}
impl std::error::Error for RawError {}
impl From<RawError> for common_assets::Error {
fn from(e: RawError) -> Self {
Self::Conversion(Box::new(e))
}
}
impl common_assets::Asset for RawManifest {
type Loader = common_assets::RonLoader;
const EXTENSION: &'static str = LANG_EXTENSION;
}
impl common_assets::Asset for RawFragment {
type Loader = common_assets::RonLoader;
const EXTENSION: &'static str = LANG_EXTENSION;
}

56
voxygen/i18n/src/verification.rs
View File

@ -1,34 +1,7 @@
use ron::de::from_reader;
use std::{fs, path::Path};
use std::{path::Path};
use crate::data::{i18n_directories, LocalizationFragment, LANG_MANIFEST_FILE, REFERENCE_LANG};
fn verify_localization_directory(root_dir: &Path, directory_path: &Path) {
// Walk through each file in the directory
for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() {
if let Ok(file_type) = i18n_file.file_type() {
// Skip folders and the manifest file (which does not contain the same struct we
// want to load)
if file_type.is_file() {
let full_path = i18n_file.path();
println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap());
let f = fs::File::open(&full_path).expect("Failed opening file");
let _loc: LocalizationFragment = match from_reader(f) {
Ok(v) => v,
Err(e) => {
panic!(
"Could not parse {} RON file, error: {}",
full_path.to_string_lossy(),
e
);
},
};
} else if file_type.is_dir() {
verify_localization_directory(root_dir, &i18n_file.path());
}
}
}
}
use crate::{i18n_directories, LANG_MANIFEST_FILE, REFERENCE_LANG};
use crate::raw;
/// Test to verify all languages that they are VALID and loadable, without
/// need of git just on the local assets folder
@ -36,17 +9,18 @@ fn verify_localization_directory(root_dir: &Path, directory_path: &Path) {
/// `asset_path` - relative path to asset directory (right now it is
/// 'assets/voxygen/i18n')
pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) {
let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG);
let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron");
let i18n_root_path = root_dir.join(asset_path);
let ref_i18n_path = i18n_root_path.join(REFERENCE_LANG);
let ref_i18n_manifest_path = ref_i18n_path.join(LANG_MANIFEST_FILE.to_string() + "." + crate::LANG_EXTENSION);
assert!(
root_dir.join(&ref_i18n_dir_path).is_dir(),
root_dir.join(&ref_i18n_path).is_dir(),
"Reference language folder doesn't exist, something is wrong!"
);
assert!(
root_dir.join(&ref_i18n_path).is_file(),
root_dir.join(&ref_i18n_manifest_path).is_file(),
"Reference language manifest file doesn't exist, something is wrong!"
);
let i18n_directories = i18n_directories(&root_dir.join(asset_path));
let i18n_directories = i18n_directories(&i18n_root_path);
// This simple check ONLY guarantees that an arbitrary minimum of translation
// files exists. It's just to notice unintentional deletion of all
// files, or modifying the paths. In case you want to delete all
@ -57,11 +31,19 @@ pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) {
folder is empty?"
);
for i18n_directory in i18n_directories {
let display_language_identifier = i18n_directory.strip_prefix(&root_dir).unwrap().as_os_str().to_str().unwrap();
let language_identifier = i18n_directory.strip_prefix(&i18n_root_path).unwrap().as_os_str().to_str().unwrap();
println!(
"verifying {:?}",
i18n_directory.strip_prefix(&root_dir).unwrap()
display_language_identifier
);
// Walk through each files and try to load them
verify_localization_directory(root_dir, &i18n_directory);
verify_localization_directory(root_dir, &asset_path, language_identifier);
}
}
fn verify_localization_directory(root_dir: &Path, asset_path: &Path, language_identifier: &str) {
let i18n_path = root_dir.join(asset_path);
let manifest = raw::load_manifest(&i18n_path, language_identifier).expect("error accessing manifest file");
raw::load_raw_language(&i18n_path, manifest).expect("error accessing fragment file");
}