Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'juliancoffee/decouple_localization_checks' into 'master'

Browse Source 
Extract voxygen/src/i18n.rs  to crate

See merge request veloren/veloren!2244
This commit is contained in:
Marcel 2021-05-09 16:47:25 +00:00
commit 587b48871d
24 changed files with 1465 additions and 1105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab/CI/build.gitlab-ci.yml
View File

@ -6,7 +6,7 @@ unittests:
script:
- ln -s /dockercache/cache-all target
- rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
- cargo test --package veloren-voxygen --lib test_all_localizations -- --nocapture --ignored
- cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored
- rm -r target/debug/incremental* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
- cargo test
retry:

33
Cargo.lock generated
View File

@ -5514,7 +5514,6 @@ name = "veloren-common"
version = "0.9.0"
dependencies = [
"approx 0.4.0",
"assets_manager",
"bitflags",
"criterion",
"crossbeam-channel",
@ -5523,7 +5522,6 @@ dependencies = [
"dot_vox",
"enum-iterator",
"hashbrown",
"image",
"indexmap",
"lazy_static",
"num-derive",
@ -5531,10 +5529,8 @@ dependencies = [
"ordered-float 2.1.1",
"rand 0.8.3",
"rayon",
"ron",
"roots",
"serde",
"serde_json",
"serde_repr",
"slab",
"slotmap",
@ -5548,9 +5544,22 @@ dependencies = [
"tracing-subscriber",
"uuid",
"vek",
"veloren-common-assets",
"veloren-common-base",
]
[[package]]
name = "veloren-common-assets"
version = "0.9.0"
dependencies = [
"assets_manager",
"dot_vox",
"image",
"lazy_static",
"ron",
"tracing",
]
[[package]]
name = "veloren-common-base"
version = "0.9.0"
@ -5643,6 +5652,19 @@ dependencies = [
"veloren-common-net",
]
[[package]]
name = "veloren-i18n"
version = "0.9.0"
dependencies = [
"deunicode",
"git2",
"hashbrown",
"ron",
"serde",
"tracing",
"veloren-common-assets",
]
[[package]]
name = "veloren-network"
version = "0.3.0"
@ -5795,7 +5817,6 @@ dependencies = [
"cpal",
"criterion",
"crossbeam",
"deunicode",
"directories-next",
"dispatch 0.1.4",
"dot_vox",
@ -5805,7 +5826,6 @@ dependencies = [
"gfx_device_gl",
"gfx_gl",
"gilrs",
"git2",
"glsl-include",
"glutin",
"glyph_brush",
@ -5843,6 +5863,7 @@ dependencies = [
"veloren-common-net",
"veloren-common-state",
"veloren-common-systems",
"veloren-i18n",
"veloren-server",
"veloren-voxygen-anim",
"veloren-world",

4
Cargo.toml
View File

@ -3,6 +3,7 @@ cargo-features = ["named-profiles","profile-overrides"]
[workspace]
members = [
"common",
"common/assets",
"common/base",
"common/ecs",
"common/net",
@ -18,9 +19,10 @@ members = [
"voxygen",
"voxygen/anim",
"voxygen/anim/dyn",
"voxygen/i18n",
"world",
"network",
"network/protocol"
"network/protocol",
]
# default profile for devs, fast to compile, okay enough to run, no debug information

6
common/Cargo.toml
View File

@ -39,14 +39,10 @@ uuid = { version = "0.8.1", default-features = false, features = ["serde", "v4"]
rand = "0.8"
# Assets
common-assets = {package = "veloren-common-assets", path = "assets"}
dot_vox = "4.0"
image = { version = "0.23.12", default-features = false, features = ["png"] }
# Assets
assets_manager = {version = "0.4.2", features = ["bincode", "ron", "json", "hot-reloading"]}
# Serde
ron = { version = "0.6", default-features = false }
serde_json = "1.0.50"
serde_repr = "0.1.6"
# csv export

14
common/assets/Cargo.toml Normal file
View File

@ -0,0 +1,14 @@
[package]
authors = ["juliancoffee <lightdarkdaughter@gmail.com>", "Marcel Märtens <marcel.cochem@googlemail.com>"]
edition = "2018"
name = "veloren-common-assets"
description = "Crate for game loading assets for veloren."
version = "0.9.0"
[dependencies]
lazy_static = "1.4.0"
assets_manager = {version = "0.4.2", features = ["bincode", "ron", "json", "hot-reloading"]}
ron = { version = "0.6", default-features = false }
dot_vox = "4.0"
image = { version = "0.23.12", default-features = false, features = ["png"] }
tracing = "0.1"

145
common/src/assets.rs → common/assets/src/lib.rs
View File

@ -20,7 +20,8 @@ pub use assets_manager::{
lazy_static! {
/// The HashMap where all loaded assets are stored in.
static ref ASSETS: AssetCache = AssetCache::new(&*ASSETS_PATH).unwrap();
static ref ASSETS: AssetCache =
AssetCache::new(&*ASSETS_PATH).unwrap();
}
pub fn start_hot_reloading() { ASSETS.enhance_hot_reloading(); }
@ -35,9 +36,9 @@ pub trait AssetExt: Sized + Send + Sync + 'static {
/// Function used to load assets from the filesystem or the cache.
/// Example usage:
/// ```no_run
/// use veloren_common::assets::{self, AssetExt};
/// use veloren_common_assets::{AssetExt, Image};
///
/// let my_image = assets::Image::load("core.ui.backgrounds.city").unwrap();
/// let my_image = Image::load("core.ui.backgrounds.city").unwrap();
/// ```
fn load(specifier: &str) -> Result<AssetHandle<Self>, Error>;
@ -53,9 +54,9 @@ pub trait AssetExt: Sized + Send + Sync + 'static {
/// Function used to load essential assets from the filesystem or the cache.
/// It will panic if the asset is not found. Example usage:
/// ```no_run
/// use veloren_common::assets::{self, AssetExt};
/// use veloren_common_assets::{AssetExt, Image};
///
/// let my_image = assets::Image::load_expect("core.ui.backgrounds.city");
/// let my_image = Image::load_expect("core.ui.backgrounds.city");
/// ```
#[track_caller]
fn load_expect(specifier: &str) -> AssetHandle<Self> {
@ -134,6 +135,7 @@ lazy_static! {
/// 3. Download & hopefully extract zip (`assets` next to binary)
/// 4. Running through cargo (`assets` in workspace root but not always in cwd incase you `cd voxygen && cargo r`)
/// 5. Running executable in the target dir (`assets` in workspace)
/// 6. Running tests (`assets` in workspace root)
pub static ref ASSETS_PATH: PathBuf = {
let mut paths = Vec::new();
@ -150,19 +152,23 @@ lazy_static! {
paths.push(path);
}
// 3. Working path
// 3. Root of the repository
if let Ok(path) = std::env::current_dir() {
paths.push(path);
// If we are in the root, push path
if path.join(".git").is_dir() {
paths.push(path);
} else {
// Search .git directory in parent directries
for ancestor in path.ancestors().take(10) {
if ancestor.join(".git").is_dir() {
paths.push(ancestor.to_path_buf());
break;
}
}
}
}
// 4. Cargo Workspace (e.g. local development)
// https://github.com/rust-lang/cargo/issues/3946#issuecomment-359619839
if let Ok(Ok(path)) = std::env::var("CARGO_MANIFEST_DIR").map(|s| s.parse::<PathBuf>()) {
paths.push(path.parent().unwrap().to_path_buf());
paths.push(path);
}
// 5. System paths
// 4. System paths
#[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))]
{
if let Ok(result) = std::env::var("XDG_DATA_HOME") {
@ -247,112 +253,3 @@ impl Compound for Directory {
Ok(Directory(files))
}
}
#[cfg(test)]
mod tests {
#[test]
fn test_assets_items() {
// TODO: Figure out how to get file name in error so only a single glob is
// needed
// Separated out into subsections so that error more descriptive
crate::comp::item::Item::new_from_asset_glob("common.items.armor.*")
.expect("Failed to iterate over armors.");
crate::comp::item::Item::new_from_asset_glob("common.items.boss_drops.*")
.expect("Failed to iterate over boss drops.");
crate::comp::item::Item::new_from_asset_glob("common.items.consumable.*")
.expect("Failed to iterate over consumables.");
crate::comp::item::Item::new_from_asset_glob("common.items.crafting_ing.*")
.expect("Failed to iterate over crafting ingredients.");
crate::comp::item::Item::new_from_asset_glob("common.items.crafting_tools.*")
.expect("Failed to iterate over crafting tools.");
crate::comp::item::Item::new_from_asset_glob("common.items.debug.*")
.expect("Failed to iterate over debug items.");
crate::comp::item::Item::new_from_asset_glob("common.items.flowers.*")
.expect("Failed to iterate over flower items.");
crate::comp::item::Item::new_from_asset_glob("common.items.food.*")
.expect("Failed to iterate over food items.");
crate::comp::item::Item::new_from_asset_glob("common.items.glider.*")
.expect("Failed to iterate over gliders.");
crate::comp::item::Item::new_from_asset_glob("common.items.grasses.*")
.expect("Failed to iterate over grasses.");
crate::comp::item::Item::new_from_asset_glob("common.items.lantern.*")
.expect("Failed to iterate over lanterns.");
crate::comp::item::Item::new_from_asset_glob("common.items.npc_armor.*")
.expect("Failed to iterate over npc armors.");
crate::comp::item::Item::new_from_asset_glob("common.items.npc_weapons.*")
.expect("Failed to iterate over npc weapons.");
crate::comp::item::Item::new_from_asset_glob("common.items.ore.*")
.expect("Failed to iterate over ores.");
crate::comp::item::Item::new_from_asset_glob("common.items.tag_examples.*")
.expect("Failed to iterate over tag examples.");
crate::comp::item::Item::new_from_asset_glob("common.items.testing.*")
.expect("Failed to iterate over testing items.");
crate::comp::item::Item::new_from_asset_glob("common.items.utility.*")
.expect("Failed to iterate over utility items.");
// Checks each weapon type to allow errors to be located more easily
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.axe.*")
.expect("Failed to iterate over axes.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.axe_1h.*")
.expect("Failed to iterate over 1h axes.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.bow.*")
.expect("Failed to iterate over bows.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.dagger.*")
.expect("Failed to iterate over daggers.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.empty.*")
.expect("Failed to iterate over empty.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.hammer.*")
.expect("Failed to iterate over hammers.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.hammer_1h.*")
.expect("Failed to iterate over 1h hammers.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sceptre.*")
.expect("Failed to iterate over sceptres.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.shield.*")
.expect("Failed to iterate over shields.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.staff.*")
.expect("Failed to iterate over staffs.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sword.*")
.expect("Failed to iterate over swords.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.sword_1h.*")
.expect("Failed to iterate over 1h swords.");
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.tool.*")
.expect("Failed to iterate over tools.");
// Checks all weapons should more weapons be added later
crate::comp::item::Item::new_from_asset_glob("common.items.weapons.*")
.expect("Failed to iterate over weapons.");
// Final at the end to account for a new folder being added
crate::comp::item::Item::new_from_asset_glob("common.items.*")
.expect("Failed to iterate over item folders.");
}
}

112
common/src/comp/inventory/item/mod.rs
View File

@ -407,8 +407,8 @@ impl Deref for Item {
}
impl assets::Compound for ItemDef {
fn load<S: assets_manager::source::Source>(
cache: &assets_manager::AssetCache<S>,
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, Error> {
// load from the filesystem first, but if the file doesn't exist, see if it's a
@ -853,3 +853,111 @@ impl<'a, T: ItemDesc + ?Sized> ItemDesc for &'a T {
fn tags(&self) -> &[ItemTag] { (*self).tags() }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_assets_items() {
// TODO: Figure out how to get file name in error so only a single glob is
// needed
// Separated out into subsections so that error more descriptive
Item::new_from_asset_glob("common.items.armor.*").expect("Failed to iterate over armors.");
Item::new_from_asset_glob("common.items.boss_drops.*")
.expect("Failed to iterate over boss drops.");
Item::new_from_asset_glob("common.items.consumable.*")
.expect("Failed to iterate over consumables.");
Item::new_from_asset_glob("common.items.crafting_ing.*")
.expect("Failed to iterate over crafting ingredients.");
Item::new_from_asset_glob("common.items.crafting_tools.*")
.expect("Failed to iterate over crafting tools.");
Item::new_from_asset_glob("common.items.debug.*")
.expect("Failed to iterate over debug items.");
Item::new_from_asset_glob("common.items.flowers.*")
.expect("Failed to iterate over flower items.");
Item::new_from_asset_glob("common.items.food.*")
.expect("Failed to iterate over food items.");
Item::new_from_asset_glob("common.items.glider.*")
.expect("Failed to iterate over gliders.");
Item::new_from_asset_glob("common.items.grasses.*")
.expect("Failed to iterate over grasses.");
Item::new_from_asset_glob("common.items.lantern.*")
.expect("Failed to iterate over lanterns.");
Item::new_from_asset_glob("common.items.npc_armor.*")
.expect("Failed to iterate over npc armors.");
Item::new_from_asset_glob("common.items.npc_weapons.*")
.expect("Failed to iterate over npc weapons.");
Item::new_from_asset_glob("common.items.ore.*").expect("Failed to iterate over ores.");
Item::new_from_asset_glob("common.items.tag_examples.*")
.expect("Failed to iterate over tag examples.");
Item::new_from_asset_glob("common.items.testing.*")
.expect("Failed to iterate over testing items.");
Item::new_from_asset_glob("common.items.utility.*")
.expect("Failed to iterate over utility items.");
// Checks each weapon type to allow errors to be located more easily
Item::new_from_asset_glob("common.items.weapons.axe.*")
.expect("Failed to iterate over axes.");
Item::new_from_asset_glob("common.items.weapons.axe_1h.*")
.expect("Failed to iterate over 1h axes.");
Item::new_from_asset_glob("common.items.weapons.bow.*")
.expect("Failed to iterate over bows.");
Item::new_from_asset_glob("common.items.weapons.dagger.*")
.expect("Failed to iterate over daggers.");
Item::new_from_asset_glob("common.items.weapons.empty.*")
.expect("Failed to iterate over empty.");
Item::new_from_asset_glob("common.items.weapons.hammer.*")
.expect("Failed to iterate over hammers.");
Item::new_from_asset_glob("common.items.weapons.hammer_1h.*")
.expect("Failed to iterate over 1h hammers.");
Item::new_from_asset_glob("common.items.weapons.sceptre.*")
.expect("Failed to iterate over sceptres.");
Item::new_from_asset_glob("common.items.weapons.shield.*")
.expect("Failed to iterate over shields.");
Item::new_from_asset_glob("common.items.weapons.staff.*")
.expect("Failed to iterate over staffs.");
Item::new_from_asset_glob("common.items.weapons.sword.*")
.expect("Failed to iterate over swords.");
Item::new_from_asset_glob("common.items.weapons.sword_1h.*")
.expect("Failed to iterate over 1h swords.");
Item::new_from_asset_glob("common.items.weapons.tool.*")
.expect("Failed to iterate over tools.");
// Checks all weapons should more weapons be added later
Item::new_from_asset_glob("common.items.weapons.*")
.expect("Failed to iterate over weapons.");
// Final at the end to account for a new folder being added
Item::new_from_asset_glob("common.items.*").expect("Failed to iterate over item folders.");
}
}

4
common/src/comp/inventory/item/tool.rs
View File

@ -392,8 +392,8 @@ impl Asset for AbilityMap<String> {
}
impl assets::Compound for AbilityMap {
fn load<S: assets_manager::source::Source>(
cache: &assets_manager::AssetCache<S>,
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, assets::Error> {
let manifest = cache.load::<AbilityMap<String>>(specifier)?.read();

2
common/src/comp/inventory/trade_pricing.rs
View File

@ -4,7 +4,7 @@ use crate::{
recipe::{default_recipe_book, RecipeInput},
trade::Good,
};
use assets_manager::AssetGuard;
use assets::AssetGuard;
use hashbrown::HashMap;
use lazy_static::lazy_static;
use serde::Deserialize;

2
common/src/lib.rs
View File

@ -23,7 +23,7 @@ pub use uuid;
// modules
#[cfg(not(target_arch = "wasm32"))]
pub mod assets;
pub use common_assets as assets;
#[cfg(not(target_arch = "wasm32"))] pub mod astar;
#[cfg(not(target_arch = "wasm32"))]
mod cached_spatial_grid;

6
common/src/recipe.rs
View File

@ -108,10 +108,10 @@ impl assets::Asset for RawRecipeBook {
}
impl assets::Compound for RecipeBook {
fn load<S: assets_manager::source::Source>(
cache: &assets_manager::AssetCache<S>,
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, assets_manager::Error> {
) -> Result<Self, assets::Error> {
#[inline]
fn load_item_def(spec: &(String, u32)) -> Result<(Arc<ItemDef>, u32), assets::Error> {
let def = Arc::<ItemDef>::load_cloned(&spec.0)?;

8
common/src/terrain/structure.rs
View File

@ -60,8 +60,8 @@ impl std::ops::Deref for StructuresGroup {
}
impl assets::Compound for StructuresGroup {
fn load<S: assets_manager::source::Source>(
cache: &assets_manager::AssetCache<S>,
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, Error> {
let specs = cache.load::<StructuresGroupSpec>(specifier)?.read();
@ -118,8 +118,8 @@ impl ReadVol for Structure {
}
impl assets::Compound for BaseStructure {
fn load<S: assets_manager::source::Source>(
cache: &assets_manager::AssetCache<S>,
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, Error> {
let dot_vox_data = cache.load::<DotVoxAsset>(specifier)?.read();

3
voxygen/Cargo.toml
View File

@ -42,6 +42,7 @@ common-systems = {package = "veloren-common-systems", path = "../common/systems"
common-state = {package = "veloren-common-state", path = "../common/state"}
anim = {package = "veloren-voxygen-anim", path = "anim"}
i18n = {package = "veloren-i18n", path = "i18n"}
# Graphics
gfx = "0.18.2"
@ -81,7 +82,6 @@ chrono = "0.4.9"
cpal = "0.13"
copy_dir = "0.1.2"
crossbeam = "0.8.0"
deunicode = "1.0"
# TODO: remove
directories-next = "2.0"
dot_vox = "4.0"
@ -121,7 +121,6 @@ winres = "0.1"
[dev-dependencies]
criterion = "0.3"
git2 = "0.13"
world = {package = "veloren-world", path = "../world"}
rayon = "1.5.0"

21
voxygen/i18n/Cargo.toml Normal file
View File

@ -0,0 +1,21 @@
[package]
authors = ["juliancoffee <lightdarkdaughter@gmail.com>", "Rémy Phelipot"]
edition = "2018"
name = "veloren-i18n"
description = "Crate for internalization and diagnostic of existing localizations."
version = "0.9.0"
[[bin]]
name = "i18n-check"
[dependencies]
# Assets
hashbrown = { version = "0.9", features = ["serde", "nightly"] }
common-assets = {package = "veloren-common-assets", path = "../../common/assets"}
deunicode = "1.0"
serde = { version = "1.0", features = ["derive"] }
# Diagnostic
git2 = "0.13"
ron = "0.6"
tracing = "0.1"

8
voxygen/i18n/README.md Normal file
View File

@ -0,0 +1,8 @@
# Usage
Get diagnostic for specific language <br/>
`$ cargo run --bin i18n-check -- --lang <lang_code>` <br/>
Test all languages <br/>
`$ cargo run --bin i18n-check -- --all`
Verify all directories <br/>
`$ cargo run --bin i18n-check -- --verify`

777
voxygen/i18n/src/analysis.rs Normal file
View File

@ -0,0 +1,777 @@
use ron::de::{from_bytes, from_reader};
use serde::{Deserialize, Serialize};
use std::{
fs,
path::{Path, PathBuf},
};
use hashbrown::{HashMap, HashSet};
/// The reference language, aka the more up-to-date localization data. Also the
/// default language at first startup.
const REFERENCE_LANG: &str = "en";
const LANG_MANIFEST_FILE: &str = "_manifest";
/// How a language can be described
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
struct LanguageMetadata {
/// A human friendly language name (e.g. "English (US)")
language_name: String,
/// A short text identifier for this language (e.g. "en_US")
///
/// On the opposite of `language_name` that can change freely,
/// `language_identifier` value shall be stable in time as it
/// is used by setting components to store the language
/// selected by the user.
language_identifier: String,
}
/// Store font metadata
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
struct Font {
/// Key to retrieve the font in the asset system
asset_key: String,
/// Scale ratio to resize the UI text dynamicly
scale_ratio: f32,
}
/// Store font metadata
type Fonts = HashMap<String, Font>;
/// Raw localization data, expect the strings to not be loaded here
/// However, metadata informations are correct
/// See `Localization` for more info on each attributes
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
struct RawLocalization {
sub_directories: Vec<String>,
string_map: HashMap<String, String>,
vector_map: HashMap<String, Vec<String>>,
convert_utf8_to_ascii: bool,
fonts: Fonts,
metadata: LanguageMetadata,
}
/// Store internationalization data
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Localization {
/// A list of subdirectories to lookup for localization files
sub_directories: Vec<String>,
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
vector_map: HashMap<String, Vec<String>>,
/// Whether to convert the input text encoded in UTF-8
/// into a ASCII version by using the `deunicode` crate.
convert_utf8_to_ascii: bool,
/// Font configuration is stored here
fonts: Fonts,
metadata: LanguageMetadata,
}
/// Store internationalization maps
/// These structs are meant to be merged into a Localization
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct LocalizationFragment {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
vector_map: HashMap<String, Vec<String>>,
}
impl Localization {}
impl From<RawLocalization> for Localization {
fn from(raw: RawLocalization) -> Self {
Self {
sub_directories: raw.sub_directories,
string_map: raw.string_map,
vector_map: raw.vector_map,
convert_utf8_to_ascii: raw.convert_utf8_to_ascii,
fonts: raw.fonts,
metadata: raw.metadata,
}
}
}
impl From<RawLocalization> for LocalizationFragment {
fn from(raw: RawLocalization) -> Self {
Self {
string_map: raw.string_map,
vector_map: raw.vector_map,
}
}
}
#[derive(Clone, Debug)]
struct LocalizationList(Vec<LanguageMetadata>);
/// List localization directories as a PathBuf vector
fn i18n_directories(i18n_dir: &Path) -> Vec<PathBuf> {
fs::read_dir(i18n_dir)
.unwrap()
.map(|res| res.map(|e| e.path()).unwrap())
.filter(|e| e.is_dir())
.collect()
}
#[derive(Eq, Hash, Debug, PartialEq)]
enum LocalizationState {
UpToDate,
NotFound,
Outdated,
Unknown,
Unused,
}
#[derive(Debug, PartialEq)]
struct FindLocalization {
uptodate_entries: usize,
outdated_entries: usize,
unused_entries: usize,
notfound_entries: usize,
errors: usize,
real_entry_count: usize,
}
#[derive(Debug)]
struct LocalizationEntryState {
key_line: Option<usize>,
chuck_line_range: Option<(usize, usize)>,
commit_id: Option<git2::Oid>,
state: LocalizationState,
}
impl LocalizationEntryState {
fn new() -> LocalizationEntryState {
LocalizationEntryState {
key_line: None,
chuck_line_range: None,
commit_id: None,
state: LocalizationState::Unknown,
}
}
}
/// Returns the Git blob associated with the given reference and path
fn read_file_from_path<'a>(
repo: &'a git2::Repository,
reference: &git2::Reference,
path: &std::path::Path,
) -> git2::Blob<'a> {
let tree = reference
.peel_to_tree()
.expect("Impossible to peel HEAD to a tree object");
tree.get_path(path)
.unwrap_or_else(|_| {
panic!(
"Impossible to find the file {:?} in reference {:?}",
path,
reference.name()
)
})
.to_object(&repo)
.unwrap()
.peel_to_blob()
.expect("Impossible to fetch the Git object")
}
fn correspond(line: &str, key: &str) -> bool {
let pat = {
// Get left part of split
let mut begin = line
.split(':')
.next()
.expect("split always produces value")
.trim()
.chars();
// Remove quotes
begin.next();
begin.next_back();
begin.as_str()
};
pat == key
}
fn generate_key_version<'a>(
repo: &'a git2::Repository,
localization: &LocalizationFragment,
path: &std::path::Path,
file_blob: &git2::Blob,
) -> HashMap<String, LocalizationEntryState> {
let mut keys: HashMap<String, LocalizationEntryState> = localization
.string_map
.keys()
.map(|k| (k.to_owned(), LocalizationEntryState::new()))
.collect();
let mut to_process: HashSet<&String> = localization.string_map.keys().collect();
// Find key start lines
let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file");
// Make the file hot
for (line_nb, line) in file_content.lines().enumerate() {
let mut found_key = None;
for key in to_process.iter() {
if correspond(line, key) {
found_key = Some(key.to_owned());
}
}
if let Some(key) = found_key {
keys.get_mut(key).unwrap().key_line = Some(line_nb);
to_process.remove(key);
};
}
let mut error_check_set: Vec<String> = vec![];
// Find commit for each keys
repo.blame_file(path, None)
.expect("Impossible to generate the Git blame")
.iter()
.for_each(|e: git2::BlameHunk| {
for (key, state) in keys.iter_mut() {
let line = match state.key_line {
Some(l) => l,
None => {
if !error_check_set.contains(key) {
eprintln!(
"Key {} does not have a git line in it's state! Skipping key.",
key
);
error_check_set.push(key.clone());
}
continue;
},
};
if line + 1 >= e.final_start_line()
&& line + 1 < e.final_start_line() + e.lines_in_hunk()
{
state.chuck_line_range = Some((
e.final_start_line(),
e.final_start_line() + e.lines_in_hunk(),
));
state.commit_id = match state.commit_id {
Some(existing_commit) => {
match repo.graph_descendant_of(e.final_commit_id(), existing_commit) {
Ok(true) => Some(e.final_commit_id()),
Ok(false) => Some(existing_commit),
Err(err) => panic!("{}", err),
}
},
None => Some(e.final_commit_id()),
};
}
}
});
keys
}
fn complete_key_versions<'a>(
repo: &'a git2::Repository,
head_ref: &git2::Reference,
i18n_key_versions: &mut HashMap<String, LocalizationEntryState>,
root_dir: &Path,
asset_path: &Path,
) {
//TODO: review unwraps in this file
// For each file (if it's not a directory) in directory
for i18n_file in root_dir.join(&asset_path).read_dir().unwrap().flatten() {
if let Ok(file_type) = i18n_file.file_type() {
if file_type.is_file() {
println!("-> {:?}", i18n_file.file_name());
let full_path = i18n_file.path();
let path = full_path.strip_prefix(root_dir).unwrap();
let i18n_blob = read_file_from_path(&repo, &head_ref, &path);
let i18n: LocalizationFragment = match from_bytes(i18n_blob.content()) {
Ok(v) => v,
Err(e) => {
eprintln!(
"Could not parse {} RON file, skipping: {}",
i18n_file.path().to_string_lossy(),
e
);
continue;
},
};
i18n_key_versions.extend(generate_key_version(&repo, &i18n, &path, &i18n_blob));
}
}
}
}
fn verify_localization_directory(root_dir: &Path, directory_path: &Path) {
// Walk through each file in the directory
for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() {
if let Ok(file_type) = i18n_file.file_type() {
// Skip folders and the manifest file (which does not contain the same struct we
// want to load)
if file_type.is_file()
&& i18n_file.file_name().to_string_lossy()
!= (LANG_MANIFEST_FILE.to_string() + ".ron")
{
let full_path = i18n_file.path();
println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap());
let f = fs::File::open(&full_path).expect("Failed opening file");
let _: LocalizationFragment = match from_reader(f) {
Ok(v) => v,
Err(e) => {
panic!(
"Could not parse {} RON file, error: {}",
full_path.to_string_lossy(),
e
);
},
};
}
}
}
}
/// Test to verify all languages that they are VALID and loadable, without
/// need of git just on the local assets folder
/// `root_dir` - absolute path to main repo
/// `asset_path` - relative path to asset directory (right now it is
/// 'assets/voxygen/i18n')
pub fn verify_all_localizations(root_dir: &Path, asset_path: &Path) {
let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG);
let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron");
assert!(
root_dir.join(&ref_i18n_dir_path).is_dir(),
"Reference language folder doesn't exist, something is wrong!"
);
assert!(
root_dir.join(&ref_i18n_path).is_file(),
"Reference language manifest file doesn't exist, something is wrong!"
);
let i18n_directories = i18n_directories(&root_dir.join(asset_path));
// This simple check ONLY guarantees that an arbitrary minimum of translation
// files exists. It's just to notice unintentional deletion of all
// files, or modifying the paths. In case you want to delete all
// language you have to adjust this number:
assert!(
i18n_directories.len() > 5,
"have less than 5 translation folders, arbitrary minimum check failed. Maybe the i18n \
folder is empty?"
);
for i18n_directory in i18n_directories {
// Attempt to load the manifest file
let manifest_path = i18n_directory.join(LANG_MANIFEST_FILE.to_string() + ".ron");
println!(
"verifying {:?}",
manifest_path.strip_prefix(&root_dir).unwrap()
);
let f = fs::File::open(&manifest_path).expect("Failed opening file");
let raw_localization: RawLocalization = match from_reader(f) {
Ok(v) => v,
Err(e) => {
panic!(
"Could not parse {} RON file, error: {}",
i18n_directory.to_string_lossy(),
e
);
},
};
// Walk through each files and try to load them
verify_localization_directory(root_dir, &i18n_directory);
// Walk through each subdirectories and try to load files in them
for sub_directory in raw_localization.sub_directories.iter() {
let subdir_path = &i18n_directory.join(sub_directory);
verify_localization_directory(root_dir, &subdir_path);
}
}
}
// Helper function to test localization directory
// `asset_path` - path to localization directory. Relative from root of the
// repo.
// `root_dir` - absolute path to repo
// `ref_i18n_path` - path to reference manifest
// `i18n_references` - keys from reference language
// `repo` - git object for main repo
// `head_ref` - HEAD
fn test_localization_directory(
asset_path: &Path,
root_dir: &Path,
ref_i18n_path: &Path,
i18n_references: &HashMap<String, LocalizationEntryState>,
repo: &git2::Repository,
head_ref: &git2::Reference,
) -> Option<FindLocalization> {
let relfile = asset_path.join(&(LANG_MANIFEST_FILE.to_string() + ".ron"));
if relfile == ref_i18n_path {
return None;
}
println!("\n-----------------------------------");
println!("{:?}", relfile);
println!("-----------------------------------");
// Find the localization entry state
let current_blob = read_file_from_path(&repo, &head_ref, &relfile);
let current_loc: RawLocalization = match from_bytes(current_blob.content()) {
Ok(v) => v,
Err(e) => {
eprintln!(
"Could not parse {} RON file, skipping: {}",
relfile.to_string_lossy(),
e
);
return None;
},
};
let mut current_i18n = generate_key_version(
&repo,
&LocalizationFragment::from(current_loc.clone()),
&relfile,
&current_blob,
);
// read HEAD for the fragment files
complete_key_versions(&repo, &head_ref, &mut current_i18n, root_dir, &asset_path);
// read HEAD for the subfolders
for sub_directory in current_loc.sub_directories.iter() {
let subdir_path = &asset_path.join(sub_directory);
complete_key_versions(&repo, &head_ref, &mut current_i18n, root_dir, &subdir_path);
}
for (ref_key, ref_state) in i18n_references.iter() {
match current_i18n.get_mut(ref_key) {
Some(state) => {
let commit_id = match state.commit_id {
Some(c) => c,
None => {
eprintln!(
"Commit ID of key {} in i18n file {} is missing! Skipping key.",
ref_key,
relfile.to_string_lossy()
);
continue;
},
};
let ref_commit_id = match ref_state.commit_id {
Some(c) => c,
None => {
eprintln!(
"Commit ID of key {} in reference i18n file is missing! Skipping key.",
ref_key
);
continue;
},
};
if commit_id != ref_commit_id
&& !repo
.graph_descendant_of(commit_id, ref_commit_id)
.unwrap_or(false)
{
state.state = LocalizationState::Outdated;
} else {
state.state = LocalizationState::UpToDate;
}
},
None => {
current_i18n.insert(ref_key.to_owned(), LocalizationEntryState {
key_line: None,
chuck_line_range: None,
commit_id: None,
state: LocalizationState::NotFound,
});
},
}
}
let ref_keys: HashSet<&String> = i18n_references.keys().collect();
for (_, state) in current_i18n
.iter_mut()
.filter(|&(k, _)| !ref_keys.contains(k))
{
state.state = LocalizationState::Unused;
}
let keys: Vec<&String> = current_i18n.keys().collect();
let mut state_map: HashMap<LocalizationState, Vec<(&String, Option<git2::Oid>)>> =
HashMap::new();
state_map.insert(LocalizationState::Outdated, Vec::new());
state_map.insert(LocalizationState::NotFound, Vec::new());
state_map.insert(LocalizationState::Unknown, Vec::new());
state_map.insert(LocalizationState::Unused, Vec::new());
let current_i18n_entry_count = current_i18n.len();
let mut uptodate_entries = 0;
let mut outdated_entries = 0;
let mut unused_entries = 0;
let mut notfound_entries = 0;
let mut unknown_entries = 0;
for key in keys {
let entry = current_i18n.get(key).unwrap();
if entry.state != LocalizationState::UpToDate {
let state_keys = state_map
.get_mut(&entry.state)
.expect("vectors must be added");
state_keys.push((key, entry.commit_id));
match entry.state {
LocalizationState::Outdated => outdated_entries += 1,
LocalizationState::NotFound => notfound_entries += 1,
LocalizationState::Unknown => unknown_entries += 1,
LocalizationState::Unused => unused_entries += 1,
LocalizationState::UpToDate => unreachable!(),
};
} else {
uptodate_entries += 1;
}
}
// Display
println!(
"\n{:60}| {:40} | {:40}\n",
"Key name",
relfile.to_str().unwrap(),
ref_i18n_path.to_str().unwrap()
);
for (state, mut lines) in state_map {
if lines.is_empty() {
continue;
}
println!("\n\t[{:?}]", state);
lines.sort();
for line in lines {
println!(
"{:60}| {:40} | {:40}",
line.0,
line.1
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_string()),
i18n_references
.get(line.0)
.map(|s| s.commit_id)
.flatten()
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_string()),
);
}
}
println!(
"\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries",
uptodate_entries, outdated_entries, unused_entries, notfound_entries, unknown_entries
);
// Calculate key count that actually matter for the status of the translation
// Unused entries don't break the game
let real_entry_count = current_i18n_entry_count - unused_entries;
let uptodate_percent = (uptodate_entries as f32 / real_entry_count as f32) * 100_f32;
let outdated_percent = (outdated_entries as f32 / real_entry_count as f32) * 100_f32;
let untranslated_percent =
((notfound_entries + unknown_entries) as f32 / real_entry_count as f32) * 100_f32;
println!(
"{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n",
uptodate_percent, outdated_percent, untranslated_percent,
);
let result = FindLocalization {
uptodate_entries,
unused_entries,
outdated_entries,
notfound_entries,
errors: unknown_entries,
real_entry_count,
};
Some(result)
}
/// Test one language
/// `code` - name of the directory in assets (de_DE for example)
/// `root_dir` - absolute path to main repo
/// `asset_path` - relative path to asset directory (right now it is
/// 'assets/voxygen/i18n')
pub fn test_specific_localization(code: String, root_dir: &Path, asset_path: &Path) {
// Relative paths from root of repo to assets
let ref_lang_dir = asset_path.join(REFERENCE_LANG);
let ref_manifest = ref_lang_dir.join(LANG_MANIFEST_FILE.to_string() + ".ron");
// Initialize Git objects
let repo = git2::Repository::discover(&root_dir)
.unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir));
let head_ref = repo.head().expect("Impossible to get the HEAD reference");
// Read HEAD for the reference language manifest
let ref_manifest_blob = read_file_from_path(&repo, &head_ref, &ref_manifest);
let loc: RawLocalization = from_bytes(ref_manifest_blob.content())
.expect("Expect to parse reference i18n RON file, can't proceed without it");
let mut i18n_references: HashMap<String, LocalizationEntryState> = generate_key_version(
&repo,
&LocalizationFragment::from(loc.clone()),
&ref_manifest,
&ref_manifest_blob,
);
// Gathering info about keys from reference language
complete_key_versions(
&repo,
&head_ref,
&mut i18n_references,
root_dir,
&ref_lang_dir,
);
for sub_directory in loc.sub_directories.iter() {
let subdir_path = &ref_lang_dir.join(sub_directory);
complete_key_versions(
&repo,
&head_ref,
&mut i18n_references,
root_dir,
&subdir_path,
);
}
// Testing how specific language is localized
let dir = asset_path.join(code);
test_localization_directory(
&dir,
root_dir,
&ref_manifest,
&i18n_references,
&repo,
&head_ref,
);
}
/// Test all localizations
/// `root_dir` - absolute path to main repo
/// `asset_path` - relative path to asset directory (right now it is
/// 'assets/voxygen/i18n')
pub fn test_all_localizations(root_dir: &Path, asset_path: &Path) {
let ref_i18n_dir_path = asset_path.join(REFERENCE_LANG);
let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron");
if !root_dir.join(&ref_i18n_dir_path).is_dir() {
panic!(
"Reference language folder not found {:?}",
&ref_i18n_dir_path
)
}
if !root_dir.join(&ref_i18n_path).is_file() {
panic!("Reference language file not found {:?}", &ref_i18n_path)
}
// Initialize Git objects
let repo = git2::Repository::discover(&root_dir)
.unwrap_or_else(|_| panic!("Failed to open the Git repository at {:?}", &root_dir));
let head_ref = repo.head().expect("Impossible to get the HEAD reference");
// Read HEAD for the reference language file
let i18n_ref_blob = read_file_from_path(&repo, &head_ref, &ref_i18n_path);
let loc: RawLocalization = from_bytes(i18n_ref_blob.content())
.expect("Expect to parse reference i18n RON file, can't proceed without it");
let mut i18n_references: HashMap<String, LocalizationEntryState> = generate_key_version(
&repo,
&LocalizationFragment::from(loc.clone()),
&ref_i18n_path,
&i18n_ref_blob,
);
// Gathering info about keys from reference language
complete_key_versions(
&repo,
&head_ref,
&mut i18n_references,
root_dir,
&ref_i18n_dir_path,
);
// read HEAD for the subfolders
for sub_directory in loc.sub_directories.iter() {
let subdir_path = &ref_i18n_dir_path.join(sub_directory);
complete_key_versions(
&repo,
&head_ref,
&mut i18n_references,
root_dir,
&subdir_path,
);
}
// Compare to other reference files
let i18n_directories = i18n_directories(&root_dir.join(asset_path));
let mut i18n_entry_counts: HashMap<PathBuf, FindLocalization> = HashMap::new();
for dir in &i18n_directories {
let rel_dir = dir.strip_prefix(root_dir).unwrap();
let result = test_localization_directory(
rel_dir,
root_dir,
&ref_i18n_path,
&i18n_references,
&repo,
&head_ref,
);
if let Some(values) = result {
i18n_entry_counts.insert(dir.clone(), values);
}
}
let mut overall_uptodate_entry_count = 0;
let mut overall_outdated_entry_count = 0;
let mut overall_untranslated_entry_count = 0;
let mut overall_real_entry_count = 0;
println!("-----------------------------------------------------------------------------");
println!("Overall Translation Status");
println!("-----------------------------------------------------------------------------");
println!(
"{:12}| {:8} | {:8} | {:8} | {:8} | {:8}",
"", "up-to-date", "outdated", "untranslated", "unused", "errors",
);
for (path, test_result) in i18n_entry_counts {
let FindLocalization {
uptodate_entries: uptodate,
outdated_entries: outdated,
unused_entries: unused,
notfound_entries: untranslated,
errors,
real_entry_count: real,
} = test_result;
overall_uptodate_entry_count += uptodate;
overall_outdated_entry_count += outdated;
overall_untranslated_entry_count += untranslated;
overall_real_entry_count += real;
println!(
"{:12}|{:8} |{:6} |{:8} |{:6} |{:8}",
path.file_name().unwrap().to_string_lossy(),
uptodate,
outdated,
untranslated,
unused,
errors,
);
}
println!(
"\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated",
(overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
(overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
(overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
);
println!("-----------------------------------------------------------------------------\n");
}

22
voxygen/i18n/src/bin/i18n-check.rs Normal file
View File

@ -0,0 +1,22 @@
use std::{env::args, path::Path, vec::Vec};
use veloren_i18n::analysis;
fn main() {
let cli: Vec<String> = args().collect();
// Generate paths
let curr_dir = std::env::current_dir().unwrap();
let root = curr_dir.parent().unwrap().parent().unwrap();
let asset_path = Path::new("assets/voxygen/i18n/");
for (i, arg) in cli.iter().enumerate() {
match arg.as_str() {
"--all" => analysis::test_all_localizations(root, asset_path),
"--verify" => analysis::verify_all_localizations(root, asset_path),
"--lang" => {
let code = cli[i + 1].clone();
analysis::test_specific_localization(code, root, asset_path);
},
_ => continue,
}
}
}

430
voxygen/i18n/src/data.rs Normal file
View File

@ -0,0 +1,430 @@
use crate::assets::{self, AssetExt, AssetGuard, AssetHandle};
use deunicode::deunicode;
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use tracing::warn;
/// The reference language, aka the more up-to-date localization data.
/// Also the default language at first startup.
pub const REFERENCE_LANG: &str = "en";
pub const LANG_MANIFEST_FILE: &str = "_manifest";
/// How a language can be described
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LanguageMetadata {
/// A human friendly language name (e.g. "English (US)")
pub language_name: String,
/// A short text identifier for this language (e.g. "en_US")
///
/// On the opposite of `language_name` that can change freely,
/// `language_identifier` value shall be stable in time as it
/// is used by setting components to store the language
/// selected by the user.
pub language_identifier: String,
}
/// Store font metadata
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Font {
/// Key to retrieve the font in the asset system
pub asset_key: String,
/// Scale ratio to resize the UI text dynamicly
scale_ratio: f32,
}
impl Font {
/// Scale input size to final UI size
pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
}
/// Store font metadata
pub type Fonts = HashMap<String, Font>;
/// Raw localization data, expect the strings to not be loaded here
/// However, metadata informations are correct
/// See `Language` for more info on each attributes
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
pub(crate) struct RawLocalization {
pub(crate) sub_directories: Vec<String>,
pub(crate) string_map: HashMap<String, String>,
pub(crate) vector_map: HashMap<String, Vec<String>>,
pub(crate) convert_utf8_to_ascii: bool,
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
}
/// Store internationalization data
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Language {
/// A list of subdirectories to lookup for localization files
pub(crate) sub_directories: Vec<String>,
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub(crate) string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub(crate) vector_map: HashMap<String, Vec<String>>,
/// Whether to convert the input text encoded in UTF-8
/// into a ASCII version by using the `deunicode` crate.
pub(crate) convert_utf8_to_ascii: bool,
/// Font configuration is stored here
pub(crate) fonts: Fonts,
pub(crate) metadata: LanguageMetadata,
}
/// Store internationalization maps
/// These structs are meant to be merged into a Language
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct LocalizationFragment {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub(crate) string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub(crate) vector_map: HashMap<String, Vec<String>>,
}
impl Language {
/// Get a localized text from the given key
pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> {
self.string_map.get(key).map(|s| s.as_str())
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> {
self.vector_map
.get(key)
.map(|v| {
if !v.is_empty() {
Some(v[index as usize % v.len()].as_str())
} else {
None
}
})
.flatten()
}
}
impl Default for Language {
fn default() -> Self {
Self {
sub_directories: Vec::default(),
string_map: HashMap::default(),
vector_map: HashMap::default(),
..Default::default()
}
}
}
impl From<RawLocalization> for Language {
fn from(raw: RawLocalization) -> Self {
Self {
sub_directories: raw.sub_directories,
string_map: raw.string_map,
vector_map: raw.vector_map,
convert_utf8_to_ascii: raw.convert_utf8_to_ascii,
fonts: raw.fonts,
metadata: raw.metadata,
}
}
}
impl From<RawLocalization> for LocalizationFragment {
fn from(raw: RawLocalization) -> Self {
Self {
string_map: raw.string_map,
vector_map: raw.vector_map,
}
}
}
impl assets::Asset for RawLocalization {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Asset for LocalizationFragment {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Compound for Language {
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
asset_key: &str,
) -> Result<Self, assets::Error> {
let raw = cache
.load::<RawLocalization>(&[asset_key, ".", LANG_MANIFEST_FILE].concat())?
.cloned();
let mut localization = Language::from(raw);
// Walk through files in the folder, collecting localization fragment to merge
// inside the asked_localization
for localization_asset in cache.load_dir::<LocalizationFragment>(asset_key)?.iter() {
localization
.string_map
.extend(localization_asset.read().string_map.clone());
localization
.vector_map
.extend(localization_asset.read().vector_map.clone());
}
// Use the localization's subdirectory list to load fragments from there
for sub_directory in localization.sub_directories.iter() {
for localization_asset in cache
.load_dir::<LocalizationFragment>(&[asset_key, ".", sub_directory].concat())?
.iter()
{
localization
.string_map
.extend(localization_asset.read().string_map.clone());
localization
.vector_map
.extend(localization_asset.read().vector_map.clone());
}
}
// Update the text if UTF-8 to ASCII conversion is enabled
if localization.convert_utf8_to_ascii {
for value in localization.string_map.values_mut() {
*value = deunicode(value);
}
for value in localization.vector_map.values_mut() {
*value = value.iter().map(|s| deunicode(s)).collect();
}
}
localization.metadata.language_name = deunicode(&localization.metadata.language_name);
Ok(localization)
}
}
/// the central data structure to handle localization in veloren
// inherit Copy+Clone from AssetHandle
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct LocalizationHandle {
active: AssetHandle<Language>,
fallback: Option<AssetHandle<Language>>,
pub use_english_fallback: bool,
}
// RAII guard returned from Localization::read(), resembles AssetGuard
pub struct LocalizationGuard {
active: AssetGuard<Language>,
fallback: Option<AssetGuard<Language>>,
}
// arbitrary choice to minimize changing all of veloren
pub type Localization = LocalizationGuard;
impl LocalizationGuard {
/// Get a localized text from the given key
///
/// First lookup is done in the active language, second in
/// the fallback (if present).
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get<'a>(&'a self, key: &'a str) -> &str {
self.active.get(key).unwrap_or_else(|| {
self.fallback
.as_ref()
.map(|f| f.get(key))
.flatten()
.unwrap_or(key)
})
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str {
self.active.get_variation(key, index).unwrap_or_else(|| {
self.fallback
.as_ref()
.map(|f| f.get_variation(key, index))
.flatten()
.unwrap_or(key)
})
}
/// Return the missing keys compared to the reference language
fn list_missing_entries(&self) -> (HashSet<String>, HashSet<String>) {
if let Some(ref_lang) = &self.fallback {
let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect();
let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect();
let strings = reference_string_keys
.difference(&string_keys)
.cloned()
.collect();
let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect();
let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect();
let vectors = reference_vector_keys
.difference(&vector_keys)
.cloned()
.collect();
(strings, vectors)
} else {
(HashSet::default(), HashSet::default())
}
}
/// Log missing entries (compared to the reference language) as warnings
pub fn log_missing_entries(&self) {
let (missing_strings, missing_vectors) = self.list_missing_entries();
for missing_key in missing_strings {
warn!(
"[{:?}] Missing string key {:?}",
self.metadata().language_identifier,
missing_key
);
}
for missing_key in missing_vectors {
warn!(
"[{:?}] Missing vector key {:?}",
self.metadata().language_identifier,
missing_key
);
}
}
pub fn fonts(&self) -> &Fonts { &self.active.fonts }
pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
}
impl LocalizationHandle {
pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
self.use_english_fallback = use_english_fallback;
}
pub fn read(&self) -> LocalizationGuard {
LocalizationGuard {
active: self.active.read(),
fallback: if self.use_english_fallback {
self.fallback.map(|f| f.read())
} else {
None
},
}
}
pub fn load(specifier: &str) -> Result<Self, crate::assets::Error> {
let default_key = ["voxygen.i18n.", REFERENCE_LANG].concat();
let language_key = ["voxygen.i18n.", specifier].concat();
let is_default = language_key == default_key;
Ok(Self {
active: Language::load(&language_key)?,
fallback: if is_default {
None
} else {
Language::load(&default_key).ok()
},
use_english_fallback: false,
})
}
pub fn load_expect(specifier: &str) -> Self {
Self::load(specifier).expect("Can't load language files")
}
pub fn reloaded(&mut self) -> bool { self.active.reloaded() }
}
#[derive(Clone, Debug)]
struct LocalizationList(Vec<LanguageMetadata>);
impl assets::Compound for LocalizationList {
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, assets::Error> {
// List language directories
let mut languages = vec![];
let i18n_root = assets::path_of(specifier, "");
for i18n_entry in (std::fs::read_dir(&i18n_root)?).flatten() {
if let Some(i18n_key) = i18n_entry.file_name().to_str() {
// load the root file of all the subdirectories
if let Ok(localization) = cache.load::<RawLocalization>(
&[specifier, ".", i18n_key, ".", LANG_MANIFEST_FILE].concat(),
) {
languages.push(localization.read().metadata.clone());
}
}
}
Ok(LocalizationList(languages))
}
}
/// Load all the available languages located in the voxygen asset directory
pub fn list_localizations() -> Vec<LanguageMetadata> {
LocalizationList::load_expect_cloned("voxygen.i18n").0
}
/// Start hot reloading of i18n assets
pub fn start_hot_reloading() { assets::start_hot_reloading(); }
#[cfg(test)]
mod tests {
use super::*;
use crate::analysis;
use std::path::Path;
// Test that localization list is loaded (not empty)
#[test]
fn test_localization_list() {
let list = list_localizations();
assert!(!list.is_empty());
}
// Test that reference language can be loaded
#[test]
fn test_localization_handle() { let _ = LocalizationHandle::load_expect(REFERENCE_LANG); }
// Test to verify all languages that they are VALID and loadable, without
// need of git just on the local assets folder
#[test]
fn verify_all_localizations() {
// Generate paths
let i18n_asset_path = Path::new("assets/voxygen/i18n/");
let curr_dir = std::env::current_dir().unwrap();
let root_dir = curr_dir.parent().unwrap().parent().unwrap();
analysis::verify_all_localizations(&root_dir, &i18n_asset_path);
}
// Test to verify all languages and print missing and faulty localisation
#[test]
#[ignore]
fn test_all_localizations() {
// Generate paths
let i18n_asset_path = Path::new("assets/voxygen/i18n/");
let curr_dir = std::env::current_dir().unwrap();
let root_dir = curr_dir.parent().unwrap().parent().unwrap();
analysis::test_all_localizations(&root_dir, &i18n_asset_path);
}
}

5
voxygen/i18n/src/lib.rs Normal file
View File

@ -0,0 +1,5 @@
pub mod analysis;
mod data;
use common_assets as assets;
pub use data::*;

940
voxygen/src/i18n.rs
View File

@ -1,940 +0,0 @@
use common::assets::{self, AssetExt, AssetGuard, AssetHandle};
use deunicode::deunicode;
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use tracing::warn;
/// The reference language, aka the more up-to-date localization data.
/// Also the default language at first startup.
pub const REFERENCE_LANG: &str = "en";
pub const LANG_MANIFEST_FILE: &str = "_manifest";
/// How a language can be described
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LanguageMetadata {
/// A human friendly language name (e.g. "English (US)")
pub language_name: String,
/// A short text identifier for this language (e.g. "en_US")
///
/// On the opposite of `language_name` that can change freely,
/// `language_identifier` value shall be stable in time as it
/// is used by setting components to store the language
/// selected by the user.
pub language_identifier: String,
}
/// Store font metadata
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Font {
/// Key to retrieve the font in the asset system
pub asset_key: String,
/// Scale ratio to resize the UI text dynamicly
pub scale_ratio: f32,
}
impl Font {
/// Scale input size to final UI size
pub fn scale(&self, value: u32) -> u32 { (value as f32 * self.scale_ratio).round() as u32 }
}
/// Store font metadata
pub type Fonts = HashMap<String, Font>;
/// Raw localization data, expect the strings to not be loaded here
/// However, metadata informations are correct
/// See `Language` for more info on each attributes
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
pub struct RawLocalization {
pub sub_directories: Vec<String>,
pub string_map: HashMap<String, String>,
pub vector_map: HashMap<String, Vec<String>>,
pub convert_utf8_to_ascii: bool,
pub fonts: Fonts,
pub metadata: LanguageMetadata,
}
/// Store internationalization data
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Language {
/// A list of subdirectories to lookup for localization files
pub sub_directories: Vec<String>,
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub vector_map: HashMap<String, Vec<String>>,
/// Whether to convert the input text encoded in UTF-8
/// into a ASCII version by using the `deunicode` crate.
pub convert_utf8_to_ascii: bool,
/// Font configuration is stored here
pub fonts: Fonts,
pub metadata: LanguageMetadata,
}
/// Store internationalization maps
/// These structs are meant to be merged into a Language
#[derive(Debug, PartialEq, Serialize, Deserialize)]
pub struct LocalizationFragment {
/// A map storing the localized texts
///
/// Localized content can be accessed using a String key.
pub string_map: HashMap<String, String>,
/// A map for storing variations of localized texts, for example multiple
/// ways of saying "Help, I'm under attack". Used primarily for npc
/// dialogue.
pub vector_map: HashMap<String, Vec<String>>,
}
impl Language {
/// Get a localized text from the given key
pub fn get<'a>(&'a self, key: &'a str) -> Option<&str> {
self.string_map.get(key).map(|s| s.as_str())
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> Option<&str> {
self.vector_map
.get(key)
.map(|v| {
if !v.is_empty() {
Some(v[index as usize % v.len()].as_str())
} else {
None
}
})
.flatten()
}
}
impl Default for Language {
fn default() -> Self {
Self {
sub_directories: Vec::default(),
string_map: HashMap::default(),
vector_map: HashMap::default(),
..Default::default()
}
}
}
impl From<RawLocalization> for Language {
fn from(raw: RawLocalization) -> Self {
Self {
sub_directories: raw.sub_directories,
string_map: raw.string_map,
vector_map: raw.vector_map,
convert_utf8_to_ascii: raw.convert_utf8_to_ascii,
fonts: raw.fonts,
metadata: raw.metadata,
}
}
}
impl From<RawLocalization> for LocalizationFragment {
fn from(raw: RawLocalization) -> Self {
Self {
string_map: raw.string_map,
vector_map: raw.vector_map,
}
}
}
impl assets::Asset for RawLocalization {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Asset for LocalizationFragment {
type Loader = assets::RonLoader;
const EXTENSION: &'static str = "ron";
}
impl assets::Compound for Language {
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
asset_key: &str,
) -> Result<Self, assets::Error> {
let raw = cache
.load::<RawLocalization>(&[asset_key, ".", LANG_MANIFEST_FILE].concat())?
.cloned();
let mut localization = Language::from(raw);
// Walk through files in the folder, collecting localization fragment to merge
// inside the asked_localization
for localization_asset in cache.load_dir::<LocalizationFragment>(asset_key)?.iter() {
localization
.string_map
.extend(localization_asset.read().string_map.clone());
localization
.vector_map
.extend(localization_asset.read().vector_map.clone());
}
// Use the localization's subdirectory list to load fragments from there
for sub_directory in localization.sub_directories.iter() {
for localization_asset in cache
.load_dir::<LocalizationFragment>(&[asset_key, ".", sub_directory].concat())?
.iter()
{
localization
.string_map
.extend(localization_asset.read().string_map.clone());
localization
.vector_map
.extend(localization_asset.read().vector_map.clone());
}
}
// Update the text if UTF-8 to ASCII conversion is enabled
if localization.convert_utf8_to_ascii {
for value in localization.string_map.values_mut() {
*value = deunicode(value);
}
for value in localization.vector_map.values_mut() {
*value = value.iter().map(|s| deunicode(s)).collect();
}
}
localization.metadata.language_name = deunicode(&localization.metadata.language_name);
Ok(localization)
}
}
/// the central data structure to handle localization in veloren
// inherit Copy+Clone from AssetHandle
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct LocalizationHandle {
active: AssetHandle<Language>,
fallback: Option<AssetHandle<Language>>,
pub use_english_fallback: bool,
}
// RAII guard returned from Localization::read(), resembles AssetGuard
pub struct LocalizationGuard {
active: AssetGuard<Language>,
fallback: Option<AssetGuard<Language>>,
}
// arbitrary choice to minimize changing all of veloren
pub type Localization = LocalizationGuard;
impl LocalizationGuard {
/// Get a localized text from the given key
///
/// First lookup is done in the active language, second in
/// the fallback (if present).
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get<'a>(&'a self, key: &'a str) -> &str {
self.active.get(key).unwrap_or_else(|| {
self.fallback
.as_ref()
.map(|f| f.get(key))
.flatten()
.unwrap_or(key)
})
}
/// Get a variation of localized text from the given key
///
/// `index` should be a random number from `0` to `u16::max()`
///
/// If the key is not present in the localization object
/// then the key is returned.
pub fn get_variation<'a>(&'a self, key: &'a str, index: u16) -> &str {
self.active.get_variation(key, index).unwrap_or_else(|| {
self.fallback
.as_ref()
.map(|f| f.get_variation(key, index))
.flatten()
.unwrap_or(key)
})
}
/// Return the missing keys compared to the reference language
fn list_missing_entries(&self) -> (HashSet<String>, HashSet<String>) {
if let Some(ref_lang) = &self.fallback {
let reference_string_keys: HashSet<_> = ref_lang.string_map.keys().cloned().collect();
let string_keys: HashSet<_> = self.active.string_map.keys().cloned().collect();
let strings = reference_string_keys
.difference(&string_keys)
.cloned()
.collect();
let reference_vector_keys: HashSet<_> = ref_lang.vector_map.keys().cloned().collect();
let vector_keys: HashSet<_> = self.active.vector_map.keys().cloned().collect();
let vectors = reference_vector_keys
.difference(&vector_keys)
.cloned()
.collect();
(strings, vectors)
} else {
(HashSet::default(), HashSet::default())
}
}
/// Log missing entries (compared to the reference language) as warnings
pub fn log_missing_entries(&self) {
let (missing_strings, missing_vectors) = self.list_missing_entries();
for missing_key in missing_strings {
warn!(
"[{:?}] Missing string key {:?}",
self.metadata().language_identifier,
missing_key
);
}
for missing_key in missing_vectors {
warn!(
"[{:?}] Missing vector key {:?}",
self.metadata().language_identifier,
missing_key
);
}
}
pub fn fonts(&self) -> &Fonts { &self.active.fonts }
pub fn metadata(&self) -> &LanguageMetadata { &self.active.metadata }
}
impl LocalizationHandle {
pub fn set_english_fallback(&mut self, use_english_fallback: bool) {
self.use_english_fallback = use_english_fallback;
}
pub fn read(&self) -> LocalizationGuard {
LocalizationGuard {
active: self.active.read(),
fallback: if self.use_english_fallback {
self.fallback.map(|f| f.read())
} else {
None
},
}
}
pub fn load(specifier: &str) -> Result<Self, common::assets::Error> {
let default_key = i18n_asset_key(REFERENCE_LANG);
let is_default = specifier == default_key;
Ok(Self {
active: Language::load(specifier)?,
fallback: if is_default {
None
} else {
Language::load(&default_key).ok()
},
use_english_fallback: false,
})
}
pub fn load_expect(specifier: &str) -> Self {
Self::load(specifier).expect("Can't load language files")
}
pub fn reloaded(&mut self) -> bool { self.active.reloaded() }
}
#[derive(Clone, Debug)]
struct LocalizationList(Vec<LanguageMetadata>);
impl assets::Compound for LocalizationList {
fn load<S: assets::source::Source>(
cache: &assets::AssetCache<S>,
specifier: &str,
) -> Result<Self, assets::Error> {
// List language directories
let mut languages = vec![];
let i18n_root = assets::path_of(specifier, "");
for i18n_entry in (std::fs::read_dir(&i18n_root)?).flatten() {
if let Some(i18n_key) = i18n_entry.file_name().to_str() {
// load the root file of all the subdirectories
if let Ok(localization) = cache.load::<RawLocalization>(
&[specifier, ".", i18n_key, ".", LANG_MANIFEST_FILE].concat(),
) {
languages.push(localization.read().metadata.clone());
}
}
}
Ok(LocalizationList(languages))
}
}
/// Load all the available languages located in the voxygen asset directory
pub fn list_localizations() -> Vec<LanguageMetadata> {
LocalizationList::load_expect_cloned("voxygen.i18n").0
}
/// Return the asset associated with the language_id
pub fn i18n_asset_key(language_id: &str) -> String { ["voxygen.i18n.", language_id].concat() }
#[cfg(test)]
mod tests {
use super::{LocalizationFragment, RawLocalization, LANG_MANIFEST_FILE, REFERENCE_LANG};
use git2::Repository;
use hashbrown::{HashMap, HashSet};
use ron::de::{from_bytes, from_reader};
use std::{
fs,
path::{Path, PathBuf},
};
/// List localization directories as a PathBuf vector
fn i18n_directories(i18n_dir: &Path) -> Vec<PathBuf> {
fs::read_dir(i18n_dir)
.unwrap()
.map(|res| res.map(|e| e.path()).unwrap())
.filter(|e| e.is_dir())
.collect()
}
#[derive(Debug, PartialEq)]
enum LocalizationState {
UpToDate,
NotFound,
Outdated,
Unknown,
Unused,
}
#[derive(Debug)]
struct LocalizationEntryState {
pub key_line: Option<usize>,
pub chuck_line_range: Option<(usize, usize)>,
pub commit_id: Option<git2::Oid>,
pub state: LocalizationState,
}
impl LocalizationEntryState {
pub fn new() -> LocalizationEntryState {
LocalizationEntryState {
key_line: None,
chuck_line_range: None,
commit_id: None,
state: LocalizationState::Unknown,
}
}
}
/// Returns the Git blob associated with the given reference and path
#[allow(clippy::expect_fun_call)] // TODO: Pending review in #587
fn read_file_from_path<'a>(
repo: &'a git2::Repository,
reference: &git2::Reference,
path: &std::path::Path,
) -> git2::Blob<'a> {
let tree = reference
.peel_to_tree()
.expect("Impossible to peel HEAD to a tree object");
tree.get_path(path)
.expect(&format!(
"Impossible to find the file {:?} in reference {:?}",
path,
reference.name()
))
.to_object(&repo)
.unwrap()
.peel_to_blob()
.expect("Impossible to fetch the Git object")
}
fn correspond(line: &str, key: &str) -> bool {
let pat = {
// Get left part of split
let mut begin = line
.split(':')
.next()
.expect("split always produces value")
.trim()
.chars();
// Remove quotes
begin.next();
begin.next_back();
begin.as_str()
};
pat == key
}
fn generate_key_version<'a>(
repo: &'a git2::Repository,
localization: &LocalizationFragment,
path: &std::path::Path,
file_blob: &git2::Blob,
) -> HashMap<String, LocalizationEntryState> {
let mut keys: HashMap<String, LocalizationEntryState> = localization
.string_map
.keys()
.map(|k| (k.to_owned(), LocalizationEntryState::new()))
.collect();
let mut to_process: HashSet<&String> = localization.string_map.keys().collect();
// Find key start lines
let file_content = std::str::from_utf8(file_blob.content()).expect("Got non UTF-8 file");
for (line_nb, line) in file_content.lines().enumerate() {
let mut found_key = None;
for key in to_process.iter() {
if correspond(line, key) {
found_key = Some(key.to_owned());
}
}
if let Some(key) = found_key {
keys.get_mut(key).unwrap().key_line = Some(line_nb);
to_process.remove(key);
};
}
let mut error_check_set: Vec<String> = vec![];
// Find commit for each keys
repo.blame_file(path, None)
.expect("Impossible to generate the Git blame")
.iter()
.for_each(|e: git2::BlameHunk| {
for (key, state) in keys.iter_mut() {
let line = match state.key_line {
Some(l) => l,
None => {
if !error_check_set.contains(key) {
eprintln!(
"Key {} does not have a git line in it's state! Skipping key.",
key
);
error_check_set.push(key.clone());
}
continue;
},
};
if line + 1 >= e.final_start_line()
&& line + 1 < e.final_start_line() + e.lines_in_hunk()
{
state.chuck_line_range = Some((
e.final_start_line(),
e.final_start_line() + e.lines_in_hunk(),
));
state.commit_id = match state.commit_id {
Some(existing_commit) => {
match repo.graph_descendant_of(e.final_commit_id(), existing_commit)
{
Ok(true) => Some(e.final_commit_id()),
Ok(false) => Some(existing_commit),
Err(err) => panic!("{}", err),
}
},
None => Some(e.final_commit_id()),
};
}
}
});
keys
}
fn complete_key_versions<'a>(
repo: &'a git2::Repository,
head_ref: &git2::Reference,
i18n_key_versions: &mut HashMap<String, LocalizationEntryState>,
dir: &Path,
) {
let root_dir = std::env::current_dir()
.map(|p| p.parent().expect("").to_owned())
.unwrap();
//TODO: review unwraps in this file
for i18n_file in root_dir.join(&dir).read_dir().unwrap().flatten() {
if let Ok(file_type) = i18n_file.file_type() {
if file_type.is_file() {
let full_path = i18n_file.path();
let path = full_path.strip_prefix(&root_dir).unwrap();
println!("-> {:?}", i18n_file.file_name());
let i18n_blob = read_file_from_path(&repo, &head_ref, &path);
let i18n: LocalizationFragment = match from_bytes(i18n_blob.content()) {
Ok(v) => v,
Err(e) => {
eprintln!(
"Could not parse {} RON file, skipping: {}",
i18n_file.path().to_string_lossy(),
e
);
continue;
},
};
i18n_key_versions.extend(generate_key_version(&repo, &i18n, &path, &i18n_blob));
}
}
}
}
fn verify_localization_directory(directory_path: &Path) {
let root_dir = std::env::current_dir()
.map(|p| p.parent().expect("").to_owned())
.unwrap();
// Walk through each file in the directory
for i18n_file in root_dir.join(&directory_path).read_dir().unwrap().flatten() {
if let Ok(file_type) = i18n_file.file_type() {
// Skip folders and the manifest file (which does not contain the same struct we
// want to load)
if file_type.is_file()
&& i18n_file.file_name().to_string_lossy()
!= (LANG_MANIFEST_FILE.to_string() + ".ron")
{
let full_path = i18n_file.path();
println!("-> {:?}", full_path.strip_prefix(&root_dir).unwrap());
let f = fs::File::open(&full_path).expect("Failed opening file");
let _: LocalizationFragment = match from_reader(f) {
Ok(v) => v,
Err(e) => {
panic!(
"Could not parse {} RON file, error: {}",
full_path.to_string_lossy(),
e
);
},
};
}
}
}
}
// Test to verify all languages that they are VALID and loadable, without
// need of git just on the local assets folder
#[test]
fn verify_all_localizations() {
// Generate paths
let i18n_asset_path = Path::new("assets/voxygen/i18n/");
let ref_i18n_dir_path = i18n_asset_path.join(REFERENCE_LANG);
let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron");
let root_dir = std::env::current_dir()
.map(|p| p.parent().expect("").to_owned())
.unwrap();
assert!(
root_dir.join(&ref_i18n_dir_path).is_dir(),
"Reference language folder doesn't exist, something is wrong!"
);
assert!(
root_dir.join(&ref_i18n_path).is_file(),
"Reference language manifest file doesn't exist, something is wrong!"
);
let i18n_directories = i18n_directories(&root_dir.join(i18n_asset_path));
// This simple check ONLY guarantees that an arbitrary minimum of translation
// files exists. It's just to notice unintentional deletion of all
// files, or modifying the paths. In case you want to delete all
// language you have to adjust this number:
assert!(
i18n_directories.len() > 5,
"have less than 5 translation folders, arbitrary minimum check failed. Maybe the i18n \
folder is empty?"
);
for i18n_directory in i18n_directories {
// Attempt to load the manifest file
let manifest_path = i18n_directory.join(LANG_MANIFEST_FILE.to_string() + ".ron");
println!(
"verifying {:?}",
manifest_path.strip_prefix(&root_dir).unwrap()
);
let f = fs::File::open(&manifest_path).expect("Failed opening file");
let raw_localization: RawLocalization = match from_reader(f) {
Ok(v) => v,
Err(e) => {
panic!(
"Could not parse {} RON file, error: {}",
i18n_directory.to_string_lossy(),
e
);
},
};
// Walk through each files and try to load them
verify_localization_directory(&i18n_directory);
// Walk through each subdirectories and try to load files in them
for sub_directory in raw_localization.sub_directories.iter() {
let subdir_path = &i18n_directory.join(sub_directory);
verify_localization_directory(&subdir_path);
}
}
}
// Test to verify all languages and print missing and faulty localisation
#[test]
#[ignore]
#[allow(clippy::expect_fun_call)]
fn test_all_localizations() {
// Generate paths
let i18n_asset_path = Path::new("assets/voxygen/i18n/");
let ref_i18n_dir_path = i18n_asset_path.join(REFERENCE_LANG);
let ref_i18n_path = ref_i18n_dir_path.join(LANG_MANIFEST_FILE.to_string() + ".ron");
let root_dir = std::env::current_dir()
.map(|p| p.parent().expect("").to_owned())
.unwrap();
let i18n_path = root_dir.join(i18n_asset_path);
if !root_dir.join(&ref_i18n_dir_path).is_dir() {
panic!(
"Reference language folder not found {:?}",
&ref_i18n_dir_path
)
}
if !root_dir.join(&ref_i18n_path).is_file() {
panic!("Reference language file not found {:?}", &ref_i18n_path)
}
// Initialize Git objects
let repo = Repository::discover(&root_dir).expect(&format!(
"Failed to open the Git repository at {:?}",
&root_dir
));
let head_ref = repo.head().expect("Impossible to get the HEAD reference");
// Read HEAD for the reference language file
let i18n_ref_blob = read_file_from_path(&repo, &head_ref, &ref_i18n_path);
let loc: RawLocalization = from_bytes(i18n_ref_blob.content())
.expect("Expect to parse reference i18n RON file, can't proceed without it");
let mut i18n_references: HashMap<String, LocalizationEntryState> = generate_key_version(
&repo,
&LocalizationFragment::from(loc.clone()),
&ref_i18n_path,
&i18n_ref_blob,
);
// read HEAD for the fragment files
complete_key_versions(&repo, &head_ref, &mut i18n_references, &ref_i18n_dir_path);
// read HEAD for the subfolders
for sub_directory in loc.sub_directories.iter() {
let subdir_path = &ref_i18n_dir_path.join(sub_directory);
complete_key_versions(&repo, &head_ref, &mut i18n_references, &subdir_path);
}
// Compare to other reference files
let i18n_directories = i18n_directories(&i18n_path);
let mut i18n_entry_counts: HashMap<PathBuf, (usize, usize, usize, usize)> = HashMap::new();
for file in &i18n_directories {
let reldir = file.strip_prefix(&root_dir).unwrap();
let relfile = reldir.join(&(LANG_MANIFEST_FILE.to_string() + ".ron"));
if relfile == ref_i18n_path {
continue;
}
println!("\n-----------------------------------");
println!("{:?}", relfile);
println!("-----------------------------------");
// Find the localization entry state
let current_blob = read_file_from_path(&repo, &head_ref, &relfile);
let current_loc: RawLocalization = match from_bytes(current_blob.content()) {
Ok(v) => v,
Err(e) => {
eprintln!(
"Could not parse {} RON file, skipping: {}",
relfile.to_string_lossy(),
e
);
continue;
},
};
let mut current_i18n = generate_key_version(
&repo,
&LocalizationFragment::from(current_loc.clone()),
&relfile,
&current_blob,
);
// read HEAD for the fragment files
complete_key_versions(&repo, &head_ref, &mut current_i18n, &reldir);
// read HEAD for the subfolders
for sub_directory in current_loc.sub_directories.iter() {
let subdir_path = &reldir.join(sub_directory);
complete_key_versions(&repo, &head_ref, &mut current_i18n, &subdir_path);
}
for (ref_key, ref_state) in i18n_references.iter() {
match current_i18n.get_mut(ref_key) {
Some(state) => {
let commit_id = match state.commit_id {
Some(c) => c,
None => {
eprintln!(
"Commit ID of key {} in i18n file {} is missing! Skipping key.",
ref_key,
relfile.to_string_lossy()
);
continue;
},
};
let ref_commit_id = match ref_state.commit_id {
Some(c) => c,
None => {
eprintln!(
"Commit ID of key {} in reference i18n file is missing! \
Skipping key.",
ref_key
);
continue;
},
};
if commit_id != ref_commit_id
&& !repo
.graph_descendant_of(commit_id, ref_commit_id)
.unwrap_or(false)
{
state.state = LocalizationState::Outdated;
} else {
state.state = LocalizationState::UpToDate;
}
},
None => {
current_i18n.insert(ref_key.to_owned(), LocalizationEntryState {
key_line: None,
chuck_line_range: None,
commit_id: None,
state: LocalizationState::NotFound,
});
},
}
}
let ref_keys: HashSet<&String> = i18n_references.keys().collect();
for (_, state) in current_i18n
.iter_mut()
.filter(|&(k, _)| !ref_keys.contains(k))
{
state.state = LocalizationState::Unused;
}
// Display
println!(
"\n{:10} | {:60}| {:40} | {:40}\n",
"State",
"Key name",
relfile.to_str().unwrap(),
ref_i18n_path.to_str().unwrap()
);
let mut sorted_keys: Vec<&String> = current_i18n.keys().collect();
sorted_keys.sort();
let current_i18n_entry_count = current_i18n.len();
let mut uptodate_entries = 0;
let mut outdated_entries = 0;
let mut unused_entries = 0;
let mut notfound_entries = 0;
let mut unknown_entries = 0;
for key in sorted_keys {
let state = current_i18n.get(key).unwrap();
if state.state != LocalizationState::UpToDate {
match state.state {
LocalizationState::Outdated => outdated_entries += 1,
LocalizationState::NotFound => notfound_entries += 1,
LocalizationState::Unknown => unknown_entries += 1,
LocalizationState::Unused => unused_entries += 1,
LocalizationState::UpToDate => unreachable!(),
};
println!(
"[{:9}] | {:60}| {:40} | {:40}",
format!("{:?}", state.state),
key,
state
.commit_id
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_string()),
i18n_references
.get(key)
.map(|s| s.commit_id)
.flatten()
.map(|s| format!("{}", s))
.unwrap_or_else(|| "None".to_string()),
);
} else {
uptodate_entries += 1;
}
}
println!(
"\n{} up-to-date, {} outdated, {} unused, {} not found, {} unknown entries",
uptodate_entries,
outdated_entries,
unused_entries,
notfound_entries,
unknown_entries
);
// Calculate key count that actually matter for the status of the translation
// Unused entries don't break the game
let real_entry_count = current_i18n_entry_count - unused_entries;
let uptodate_percent = (uptodate_entries as f32 / real_entry_count as f32) * 100_f32;
let outdated_percent = (outdated_entries as f32 / real_entry_count as f32) * 100_f32;
let untranslated_percent =
((notfound_entries + unknown_entries) as f32 / real_entry_count as f32) * 100_f32;
println!(
"{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated\n",
uptodate_percent, outdated_percent, untranslated_percent,
);
i18n_entry_counts.insert(
file.clone(),
(
uptodate_entries,
outdated_entries,
notfound_entries + unknown_entries,
real_entry_count,
),
);
}
let mut overall_uptodate_entry_count = 0;
let mut overall_outdated_entry_count = 0;
let mut overall_untranslated_entry_count = 0;
let mut overall_real_entry_count = 0;
println!("-----------------------------------------------------------------------------");
println!("Overall Translation Status");
println!("-----------------------------------------------------------------------------");
println!(
"{:12}| {:8} | {:8} | {:8}",
"", "up-to-date", "outdated", "untranslated"
);
for (path, (uptodate, outdated, untranslated, real)) in i18n_entry_counts {
overall_uptodate_entry_count += uptodate;
overall_outdated_entry_count += outdated;
overall_untranslated_entry_count += untranslated;
overall_real_entry_count += real;
println!(
"{:12}|{:8} |{:6} |{:8}",
path.file_name().unwrap().to_string_lossy(),
uptodate,
outdated,
untranslated
);
}
println!(
"\n{:.2}% up-to-date, {:.2}% outdated, {:.2}% untranslated",
(overall_uptodate_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
(overall_outdated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
(overall_untranslated_entry_count as f32 / overall_real_entry_count as f32) * 100_f32,
);
println!("-----------------------------------------------------------------------------\n");
}
}

4
voxygen/src/lib.rs
View File

@ -19,7 +19,6 @@ pub mod controller;
mod ecs;
pub mod error;
pub mod hud;
pub mod i18n;
pub mod key_state;
pub mod menu;
pub mod mesh;
@ -35,12 +34,12 @@ pub mod window;
// Reexports
pub use crate::error::Error;
pub use i18n;
#[cfg(feature = "singleplayer")]
use crate::singleplayer::Singleplayer;
use crate::{
audio::AudioFrontend,
i18n::LocalizationHandle,
profile::Profile,
render::Renderer,
settings::Settings,
@ -48,6 +47,7 @@ use crate::{
};
use common::clock::Clock;
use common_base::span;
use i18n::LocalizationHandle;
/// A type used to store state that is shared between all play states.
pub struct GlobalState {

9
voxygen/src/main.rs
View File

@ -5,7 +5,7 @@
use veloren_voxygen::{
audio::AudioFrontend,
i18n::{self, i18n_asset_key, LocalizationHandle},
i18n::{self, LocalizationHandle},
profile::Profile,
run,
scene::terrain::SpriteRenderContext,
@ -144,6 +144,7 @@ fn main() {
}));
assets::start_hot_reloading();
i18n::start_hot_reloading();
// Initialise watcher for animation hotreloading
#[cfg(feature = "hot-anim")]
@ -163,8 +164,8 @@ fn main() {
// Load the profile.
let profile = Profile::load();
let mut i18n = LocalizationHandle::load(&i18n_asset_key(&settings.language.selected_language))
.unwrap_or_else(|error| {
let mut i18n =
LocalizationHandle::load(&settings.language.selected_language).unwrap_or_else(|error| {
let selected_language = &settings.language.selected_language;
warn!(
?error,
@ -172,7 +173,7 @@ fn main() {
"Impossible to load language: change to the default language (English) instead.",
);
settings.language.selected_language = i18n::REFERENCE_LANG.to_owned();
LocalizationHandle::load_expect(&i18n_asset_key(&settings.language.selected_language))
LocalizationHandle::load_expect(&settings.language.selected_language)
});
i18n.read().log_missing_entries();
i18n.set_english_fallback(settings.language.use_english_fallback);

6
voxygen/src/menu/main/mod.rs
View File

@ -5,7 +5,7 @@ use super::char_selection::CharSelectionState;
#[cfg(feature = "singleplayer")]
use crate::singleplayer::Singleplayer;
use crate::{
i18n::{i18n_asset_key, Localization, LocalizationHandle},
i18n::{Localization, LocalizationHandle},
render::Renderer,
settings::Settings,
window::Event,
@ -295,9 +295,9 @@ impl PlayState for MainMenuState {
MainMenuEvent::ChangeLanguage(new_language) => {
global_state.settings.language.selected_language =
new_language.language_identifier;
global_state.i18n = LocalizationHandle::load_expect(&i18n_asset_key(
global_state.i18n = LocalizationHandle::load_expect(
&global_state.settings.language.selected_language,
));
);
global_state.i18n.read().log_missing_entries();
global_state
.i18n

7
voxygen/src/session/settings_change.rs
View File

@ -5,7 +5,7 @@ use crate::{
BarNumbers, BuffPosition, CrosshairType, Intro, PressBehavior, ScaleChange,
ShortcutNumbers, XpBar,
},
i18n::{i18n_asset_key, LanguageMetadata, LocalizationHandle},
i18n::{LanguageMetadata, LocalizationHandle},
render::RenderMode,
settings::{
AudioSettings, ControlSettings, Fps, GamepadSettings, GameplaySettings, GraphicsSettings,
@ -482,9 +482,8 @@ impl SettingsChange {
SettingsChange::Language(language_change) => match language_change {
Language::ChangeLanguage(new_language) => {
settings.language.selected_language = new_language.language_identifier;
global_state.i18n = LocalizationHandle::load_expect(&i18n_asset_key(
&settings.language.selected_language,
));
global_state.i18n =
LocalizationHandle::load_expect(&settings.language.selected_language);
global_state.i18n.read().log_missing_entries();
global_state
.i18n