Merge branch 'juliancoffee/fix-convert-to-unicode' into 'master'

Resurrect convert_utf8_to_ascii

See merge request veloren/veloren!3548

voxygen/i18n/src/lib.rs

@@ -10,6 +10,8 @@ use fluent_bundle::{bundle::FluentBundle, FluentResource};
 use intl_memoizer::concurrent::IntlLangMemoizer;
 use unic_langid::LanguageIdentifier;
 
+use deunicode::deunicode;
+
 use hashbrown::HashMap;
 use serde::{Deserialize, Serialize};
 use std::{borrow::Cow, io};
@@ -65,15 +67,9 @@ pub type Fonts = HashMap<String, Font>;
 struct Language {
     /// The bundle storing all localized texts
     pub(crate) bundle: FluentBundle<FluentResource, IntlLangMemoizer>,
-    /// Whether to convert the input text encoded in UTF-8
-    /// into a ASCII version by using the `deunicode` crate.
-    // FIXME (i18n convert_utf8_to_ascii):
-    #[allow(dead_code)]
-    convert_utf8_to_ascii: bool,
 
     /// Font configuration is stored here
     pub(crate) fonts: Fonts,
-
     pub(crate) metadata: LanguageMetadata,
 }
 
@@ -154,10 +150,22 @@ impl assets::Compound for Language {
             match cache.load(id) {
                 Ok(handle) => {
                     let source: &raw::Resource = &*handle.read();
-                    let resource =
-                        FluentResource::try_new(source.src.clone()).map_err(|(_ast, errs)| {
-                            ResourceErr::parsing_error(errs, id.to_owned(), &source.src)
-                        })?;
+                    let src = source.src.clone();
+
+                    // NOTE:
+                    // This deunicode whole file, which mean it may break if
+                    // we have non-ascii keys.
+                    // I don't consider this a problem, because having
+                    // non-ascii keys is quite exotic.
+                    let src = if convert_utf8_to_ascii {
+                        deunicode(&src)
+                    } else {
+                        src
+                    };
+
+                    let resource = FluentResource::try_new(src).map_err(|(_ast, errs)| {
+                        ResourceErr::parsing_error(errs, id.to_owned(), &source.src)
+                    })?;
 
                     bundle
                         .add_resource(resource)
@@ -172,7 +180,6 @@ impl assets::Compound for Language {
 
         Ok(Self {
             bundle,
-            convert_utf8_to_ascii,
             fonts,
             metadata,
         })

voxygen/i18n/src/raw.rs

@@ -1,12 +1,15 @@
-use crate::{Fonts, LanguageMetadata};
+use crate::{
+    assets::{loader, StringLoader},
+    Fonts, LanguageMetadata,
+};
 use serde::{Deserialize, Serialize};
 
-use std::str::FromStr;
-
 /// Localization metadata from manifest file
 /// See `Language` for more info on each attributes
 #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 pub(crate) struct Manifest {
+    /// Whether to convert the input text encoded in UTF-8
+    /// into a ASCII version by using the `deunicode` crate.
     pub(crate) convert_utf8_to_ascii: bool,
     pub(crate) fonts: Fonts,
     pub(crate) metadata: LanguageMetadata,
@@ -18,19 +21,31 @@ impl crate::assets::Asset for Manifest {
     const EXTENSION: &'static str = "ron";
 }
 
+// Newtype wrapper representing fluent resource.
+//
+// NOTE:
+// We store String, that later converted to FluentResource.
+// We can't do it at load time, because we might want to do utf8 to ascii
+// conversion and we know it only after we've loaded language manifest.
+//
+// Alternative solution is to make it hold Rc/Arc around FluentResource,
+// implement methods that give us mutable control around resource entries,
+// but doing it to eliminate Clone that happens N per programm life seems as
+// overengineering.
+//
+// N is time of fluent files, so about 20 for English and the same for target
+// localisation.
 #[derive(Clone)]
 pub(crate) struct Resource {
     pub(crate) src: String,
 }
 
-impl FromStr for Resource {
-    type Err = std::convert::Infallible;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self { src: s.to_owned() }) }
+impl From<String> for Resource {
+    fn from(src: String) -> Self { Self { src } }
 }
 
 impl crate::assets::Asset for Resource {
-    type Loader = crate::assets::loader::ParseLoader;
+    type Loader = loader::LoadFrom<String, StringLoader>;
 
     const EXTENSION: &'static str = "ftl";
 }