Extract voxygen/src/i18n.rs into own crate

- New crate (i18n), currently under voxygen directory
- Updated CI to run localization test from i18n, not from voxygen

.gitlab/CI/build.gitlab-ci.yml

@@ -6,7 +6,7 @@ unittests:
   script:
     - ln -s /dockercache/cache-all target
     - rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - cargo test --package veloren-voxygen --lib test_all_localizations -- --nocapture --ignored
+    - cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored
     - rm -r target/debug/incremental* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
     - cargo test
   retry:

Cargo.lock

@@ -5644,11 +5644,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "veloren-i18n-check"
+name = "veloren-i18n"
 version = "0.9.0"
 dependencies = [
+ "assets_manager",
+ "deunicode",
  "git2",
  "hashbrown",
+ "lazy_static",
  "ron",
  "serde",
  "tracing",
@@ -5854,7 +5857,7 @@ dependencies = [
  "veloren-common-net",
  "veloren-common-state",
  "veloren-common-systems",
- "veloren-i18n-check",
+ "veloren-i18n",
  "veloren-server",
  "veloren-voxygen-anim",
  "veloren-world",

Cargo.toml

@@ -18,7 +18,7 @@ members = [
     "voxygen",
     "voxygen/anim",
     "voxygen/anim/dyn",
-    "voxygen/i18n-check",
+    "voxygen/i18n",
     "world",
     "network",
     "network/protocol",

voxygen/Cargo.toml

@@ -42,7 +42,7 @@ common-systems = {package = "veloren-common-systems", path = "../common/systems"
 common-state = {package = "veloren-common-state", path = "../common/state"}
 
 anim = {package = "veloren-voxygen-anim", path = "anim"}
-i18n-check = {package = "veloren-i18n-check", path = "i18n-check"}
+i18n = {package = "veloren-i18n", path = "i18n"}
 
 # Graphics
 gfx = "0.18.2"

voxygen/i18n-check/src/lib.rs

@@ -1 +0,0 @@
-pub mod analysis;

voxygen/i18n/Cargo.toml

@@ -1,16 +1,20 @@
 [package]
 authors = ["juliancoffee <lightdarkdaughter@gmail.com>"]
 edition = "2018"
-name = "veloren-i18n-check"
+name = "veloren-i18n"
-description = "crate to analyze localization assets to find what needs update"
+description = "Crate for internalization and diagnostic of existing localizations."
 version = "0.9.0"
 
 [[bin]]
 name = "i18n-check"
 
 [dependencies]
+hashbrown = { version = "0.9", features = ["rayon", "serde", "nightly"] }
+lazy_static = "1.4.0"
+assets_manager = {version = "0.4.2", features = ["bincode", "ron", "json", "hot-reloading"]}
+deunicode = "1.0"
+ron = "0.6"
 serde = { version = "1.0", features = ["derive"] }
 tracing = "0.1"
-ron = "0.6"
+# Diagnostic
 git2 = "0.13"
-hashbrown = { version = "0.9", features = ["rayon", "serde", "nightly"] }

voxygen/i18n/src/assets.rs

@@ -0,0 +1,151 @@
+//! Load assets (images or voxel data) from files
+
+use lazy_static::lazy_static;
+use std::path::PathBuf;
+
+pub use assets_manager::{
+    asset::Ron,
+    loader::{
+        self, BincodeLoader, BytesLoader, JsonLoader, LoadFrom, Loader, RonLoader, StringLoader,
+    },
+    source, Asset, AssetCache, BoxedError, Compound, Error,
+};
+
+lazy_static! {
+    /// The HashMap where all loaded assets are stored in.
+    static ref ASSETS: AssetCache = AssetCache::new(&*ASSETS_PATH).unwrap();
+}
+
+pub type AssetHandle<T> = assets_manager::Handle<'static, T>;
+pub type AssetGuard<T> = assets_manager::AssetGuard<'static, T>;
+
+/// The Asset trait, which is implemented by all structures that have their data
+/// stored in the filesystem.
+pub trait AssetExt: Sized + Send + Sync + 'static {
+    /// Function used to load assets from the filesystem or the cache.
+    fn load(specifier: &str) -> Result<AssetHandle<Self>, Error>;
+
+    /// Function used to load assets from the filesystem or the cache and return
+    /// a clone.
+    fn load_cloned(specifier: &str) -> Result<Self, Error>
+    where
+        Self: Clone,
+    {
+        Self::load(specifier).map(AssetHandle::cloned)
+    }
+
+    /// Function used to load essential assets from the filesystem or the cache.
+    /// It will panic if the asset is not found.
+    #[track_caller]
+    fn load_expect(specifier: &str) -> AssetHandle<Self> {
+        Self::load(specifier).unwrap_or_else(|err| {
+            panic!(
+                "Failed loading essential asset: {} (error={:?})",
+                specifier, err
+            )
+        })
+    }
+
+    /// Function used to load essential assets from the filesystem or the cache
+    /// and return a clone. It will panic if the asset is not found.
+    #[track_caller]
+    fn load_expect_cloned(specifier: &str) -> Self
+    where
+        Self: Clone,
+    {
+        Self::load_expect(specifier).cloned()
+    }
+
+    fn load_owned(specifier: &str) -> Result<Self, Error>;
+}
+
+impl<T: Compound> AssetExt for T {
+    fn load(specifier: &str) -> Result<AssetHandle<Self>, Error> { ASSETS.load(specifier) }
+
+    fn load_owned(specifier: &str) -> Result<Self, Error> { ASSETS.load_owned(specifier) }
+}
+
+lazy_static! {
+    /// Lazy static to find and cache where the asset directory is.
+    /// Cases we need to account for:
+    /// 1. Running through airshipper (`assets` next to binary)
+    /// 2. Install with package manager and run (assets probably in `/usr/share/veloren/assets` while binary in `/usr/bin/`)
+    /// 3. Download & hopefully extract zip (`assets` next to binary)
+    /// 4. Running through cargo (`assets` in workspace root but not always in cwd incase you `cd voxygen && cargo r`)
+    /// 5. Running executable in the target dir (`assets` in workspace)
+    pub static ref ASSETS_PATH: PathBuf = {
+        let mut paths = Vec::new();
+
+        // Note: Ordering matters here!
+
+        // 1. VELOREN_ASSETS environment variable
+        if let Ok(var) = std::env::var("VELOREN_ASSETS") {
+            paths.push(var.into());
+        }
+
+        // 2. Executable path
+        if let Ok(mut path) = std::env::current_exe() {
+            path.pop();
+            paths.push(path);
+        }
+
+        // 3. Working path
+        if let Ok(path) = std::env::current_dir() {
+            paths.push(path);
+        }
+
+        // 4. Cargo Workspace (e.g. local development)
+        // https://github.com/rust-lang/cargo/issues/3946#issuecomment-359619839
+        if let Ok(Ok(path)) = std::env::var("CARGO_MANIFEST_DIR").map(|s| s.parse::<PathBuf>()) {
+            paths.push(path.parent().unwrap().to_path_buf());
+            paths.push(path);
+        }
+
+        // 5. System paths
+        #[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))]
+        {
+            if let Ok(result) = std::env::var("XDG_DATA_HOME") {
+                paths.push(format!("{}/veloren/", result).into());
+            } else if let Ok(result) = std::env::var("HOME") {
+                paths.push(format!("{}/.local/share/veloren/", result).into());
+            }
+
+            if let Ok(result) = std::env::var("XDG_DATA_DIRS") {
+                result.split(':').for_each(|x| paths.push(format!("{}/veloren/", x).into()));
+            } else {
+                // Fallback
+                let fallback_paths = vec!["/usr/local/share", "/usr/share"];
+                for fallback_path in fallback_paths {
+                    paths.push(format!("{}/veloren/", fallback_path).into());
+                }
+            }
+        }
+
+        tracing::trace!("Possible asset locations paths={:?}", paths);
+
+        for mut path in paths.clone() {
+            if !path.ends_with("assets") {
+                path = path.join("assets");
+            }
+
+            if path.is_dir() {
+                tracing::info!("Assets found path={}", path.display());
+                return path;
+            }
+        }
+
+        panic!(
+            "Asset directory not found. In attempting to find it, we searched:\n{})",
+            paths.iter().fold(String::new(), |mut a, path| {
+                a += &path.to_string_lossy();
+                a += "\n";
+                a
+            }),
+        );
+    };
+}
+
+/// Returns the actual path of the specifier with the extension.
+///
+/// For directories, give `""` as extension.
+pub fn path_of(specifier: &str, ext: &str) -> PathBuf { ASSETS.source().path_of(specifier, ext) }

voxygen/i18n/src/bin/i18n-check.rs

@@ -1,5 +1,5 @@
 use std::{env::args, path::Path, vec::Vec};
-use veloren_i18n_check::analysis;
+use veloren_i18n::analysis;
 
 fn main() {
     let cli: Vec<String> = args().collect();

voxygen/i18n/src/i18n.rs

@@ -1,4 +1,4 @@
-use common::assets::{self, AssetExt, AssetGuard, AssetHandle};
+use crate::assets::{self, AssetExt, AssetGuard, AssetHandle};
 use deunicode::deunicode;
 use hashbrown::{HashMap, HashSet};
 use serde::{Deserialize, Serialize};
@@ -332,7 +332,7 @@ impl LocalizationHandle {
         }
     }
 
-    pub fn load(specifier: &str) -> Result<Self, common::assets::Error> {
+    pub fn load(specifier: &str) -> Result<Self, crate::assets::Error> {
         let default_key = i18n_asset_key(REFERENCE_LANG);
         let is_default = specifier == default_key;
         Ok(Self {
@@ -390,7 +390,7 @@ pub fn i18n_asset_key(language_id: &str) -> String { ["voxygen.i18n.", language_
 
 #[cfg(test)]
 mod tests {
-    use i18n_check::analysis;
+    use crate::analysis;
     use std::path::Path;
 
     // Test to verify all languages that they are VALID and loadable, without
@@ -400,8 +400,8 @@ mod tests {
         // Generate paths
         let i18n_asset_path = Path::new("assets/voxygen/i18n/");
         let curr_dir = std::env::current_dir().unwrap();
-        let root = curr_dir.parent().unwrap();
+        let root_dir = curr_dir.parent().unwrap().parent().unwrap();
-        analysis::verify_all_localizations(&root, &i18n_asset_path);
+        analysis::verify_all_localizations(&root_dir, &i18n_asset_path);
     }
 
     // Test to verify all languages and print missing and faulty localisation
@@ -411,7 +411,7 @@ mod tests {
         // Generate paths
         let i18n_asset_path = Path::new("assets/voxygen/i18n/");
         let curr_dir = std::env::current_dir().unwrap();
-        let root = curr_dir.parent().unwrap();
+        let root_dir = curr_dir.parent().unwrap().parent().unwrap();
-        analysis::test_all_localizations(&root, &i18n_asset_path);
+        analysis::test_all_localizations(&root_dir, &i18n_asset_path);
     }
 }

voxygen/i18n/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod analysis;
+mod assets;
+mod i18n;
+
+pub use i18n::*;

voxygen/src/lib.rs

@@ -19,7 +19,6 @@ pub mod controller;
 mod ecs;
 pub mod error;
 pub mod hud;
-pub mod i18n;
 pub mod key_state;
 pub mod menu;
 pub mod mesh;
@@ -35,12 +34,12 @@ pub mod window;
 
 // Reexports
 pub use crate::error::Error;
+pub use i18n;
 
 #[cfg(feature = "singleplayer")]
 use crate::singleplayer::Singleplayer;
 use crate::{
     audio::AudioFrontend,
-    i18n::LocalizationHandle,
     profile::Profile,
     render::Renderer,
     settings::Settings,
@@ -48,6 +47,7 @@ use crate::{
 };
 use common::clock::Clock;
 use common_base::span;
+use i18n::LocalizationHandle;
 
 /// A type used to store state that is shared between all play states.
 pub struct GlobalState {