Move shaders, start asset reloading system

common/Cargo.toml

@@ -27,3 +27,4 @@ hashbrown = { version = "0.5.0", features = ["serde", "nightly"] }
 find_folder = "0.3.0"
 parking_lot = "0.9.0"
 crossbeam = "0.7.2"
+notify = "5.0.0-pre.1"

common/src/assets/mod.rs

@@ -1,4 +1,5 @@
 //! Load assets (images or voxel data) from files
+pub mod watch;
 
 use dot_vox::DotVoxData;
 use hashbrown::HashMap;
@@ -60,7 +61,7 @@ pub fn load_map<A: Asset + 'static, F: FnOnce(A) -> A>(
     f: F,
 ) -> Result<Arc<A>, Error> {
     let mut assets_write = ASSETS.write().unwrap();
-    match assets_write.get(&(specifier.to_owned() + A::ENDINGS[0])) {
+    match assets_write.get(specifier) {
         Some(asset) => Ok(Arc::clone(asset).downcast()?),
         None => {
             let asset = Arc::new(f(A::parse(load_file(specifier, A::ENDINGS)?)?));
@@ -95,8 +96,53 @@ pub fn load_expect<A: Asset + 'static>(specifier: &str) -> Arc<A> {
     load(specifier).unwrap_or_else(|_| panic!("Failed loading essential asset: {}", specifier))
 }
 
+/// Load an asset while registering it to be watched and reloaded when it changes
+pub fn load_watched<A: Asset + 'static>(
+    specifier: &str,
+    indicator: &mut watch::ReloadIndicator,
+) -> Result<Arc<A>, Error> {
+    // Determine path to watch
+    let mut path = unpack_specifier(specifier);
+    let mut file_exists = false;
+    for ending in A::ENDINGS {
+        let mut path = path.clone();
+        path.set_extension(ending);
+
+        if path.exists() {
+            file_exists = true;
+            break;
+        }
+    }
+    if !file_exists {
+        return Err(Error::NotFound(path.to_string_lossy().into_owned()));
+    }
+
+    // Start watching first to detect any changes while the file is being loaded
+    let owned_specifier = specifier.to_string();
+    indicator.add(path, move || {
+        // TODO: handle result
+        reload::<A>(&owned_specifier);
+    });
+    load(specifier)
+}
+
 /// The Asset trait, which is implemented by all structures that have their data stored in the
 /// filesystem.
+fn reload<A: Asset + 'static>(specifier: &str) -> Result<(), Error> {
+    let asset = Arc::new(A::parse(load_file(specifier, A::ENDINGS)?)?);
+    let clone = Arc::clone(&asset);
+    let mut assets_write = ASSETS.write().unwrap();
+    match assets_write.get_mut(specifier) {
+        Some(a) => *a = clone,
+        None => {
+            assets_write.insert(specifier.to_owned(), clone);
+        }
+    }
+
+    Ok(())
+}
+
+/// Asset Trait
 pub trait Asset: Send + Sync + Sized {
     const ENDINGS: &'static [&'static str];
     /// Parse the input file and return the correct Asset.
@@ -129,53 +175,64 @@ impl Asset for Value {
     }
 }
 
-/// Function to find where the asset/ directory is.
+impl Asset for String {
-fn assets_dir() -> PathBuf {
+    const ENDINGS: &'static [&'static str] = &["glsl"];
-    let mut paths = Vec::new();
+    fn parse(mut buf_reader: BufReader<File>) -> Result<Self, Error> {
-
+        let mut string = String::new();
-    // VELOREN_ASSETS environment variable
+        buf_reader.read_to_string(&mut string)?;
-    if let Ok(var) = std::env::var("VELOREN_ASSETS") {
+        Ok(string)
-        paths.push(var.to_owned().into());
     }
+}
 
-    // Executable path
+/// Lazy static to find and cache where the asset directory is.
-    if let Ok(mut path) = std::env::current_exe() {
+lazy_static! {
-        path.pop();
+    static ref ASSETS_PATH: PathBuf = {
-        paths.push(path);
+        let mut paths = Vec::new();
-    }
 
-    // Working path
+        // VELOREN_ASSETS environment variable
-    if let Ok(path) = std::env::current_dir() {
+        if let Ok(var) = std::env::var("VELOREN_ASSETS") {
-        paths.push(path);
+            paths.push(var.to_owned().into());
-    }
-
-    // System paths
-    #[cfg(target_os = "linux")]
-    paths.push("/usr/share/veloren/assets".into());
-
-    for path in paths.clone() {
-        match find_folder::Search::ParentsThenKids(3, 1)
-            .of(path)
-            .for_folder("assets")
-        {
-            Ok(assets_path) => return assets_path,
-            Err(_) => continue,
         }
-    }
 
-    panic!(
+        // Executable path
-        "Asset directory not found. In attempting to find it, we searched:\n{})",
+        if let Ok(mut path) = std::env::current_exe() {
-        paths.iter().fold(String::new(), |mut a, path| {
+            path.pop();
-            a += &path.to_string_lossy();
+            paths.push(path);
-            a += "\n";
+        }
-            a
+
-        }),
+        // Working path
-    );
+        if let Ok(path) = std::env::current_dir() {
+            paths.push(path);
+        }
+
+        // System paths
+        #[cfg(target_os = "linux")]
+        paths.push("/usr/share/veloren/assets".into());
+
+        for path in paths.clone() {
+            match find_folder::Search::ParentsThenKids(3, 1)
+                .of(path)
+                .for_folder("assets")
+            {
+                Ok(assets_path) => return assets_path,
+                Err(_) => continue,
+            }
+        }
+
+        panic!(
+            "Asset directory not found. In attempting to find it, we searched:\n{})",
+            paths.iter().fold(String::new(), |mut a, path| {
+                a += &path.to_string_lossy();
+                a += "\n";
+                a
+            }),
+        );
+    };
 }
 
 /// Converts a specifier like "core.backgrounds.city" to ".../veloren/assets/core/backgrounds/city".
 fn unpack_specifier(specifier: &str) -> PathBuf {
-    let mut path = assets_dir();
+    let mut path = ASSETS_PATH.clone();
     path.push(specifier.replace(".", "/"));
     path
 }

common/src/assets/watch.rs

@@ -0,0 +1,156 @@
+use crossbeam::channel::{select, unbounded, Receiver, Sender};
+use lazy_static::lazy_static;
+use log::warn;
+use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher as _};
+use std::{
+    collections::HashMap,
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, Mutex, Weak,
+    },
+    thread,
+    time::Duration,
+};
+
+type Handler = Box<dyn Fn() + Send>;
+
+lazy_static! {
+    static ref WATCHER: Mutex<Sender<(PathBuf, Handler, Weak<AtomicBool>)>> =
+        Mutex::new(Watcher::new().run());
+}
+
+// This will need to be adjusted when specifier mapping to asset location becomes more dynamic
+struct Watcher {
+    watching: HashMap<PathBuf, (Handler, Vec<Weak<AtomicBool>>)>,
+    watcher: RecommendedWatcher,
+    event_rx: Receiver<Result<Event, notify::Error>>,
+}
+impl Watcher {
+    fn new() -> Self {
+        let (event_tx, event_rx) = unbounded();
+        Watcher {
+            watching: HashMap::new(),
+            watcher: notify::Watcher::new(event_tx, Duration::from_secs(2))
+                .expect("Failed to create notify::Watcher"),
+            event_rx,
+        }
+    }
+    fn watch(&mut self, path: PathBuf, handler: Handler, signal: Weak<AtomicBool>) {
+        match self.watching.get_mut(&path) {
+            Some((_, ref mut v)) => {
+                if !v.iter().any(|s| match (s.upgrade(), signal.upgrade()) {
+                    (Some(arc1), Some(arc2)) => Arc::ptr_eq(&arc1, &arc2),
+                    _ => false,
+                }) {
+                    v.push(signal);
+                }
+            }
+            None => {
+                // TODO handle this result
+                self.watcher.watch(path.clone(), RecursiveMode::Recursive);
+                self.watching.insert(path, (handler, vec![signal]));
+            }
+        }
+    }
+    fn handle_event(&mut self, event: Event) {
+        // TODO: consider using specific modify variant
+        if let Event {
+            kind: EventKind::Modify(_),
+            paths,
+            ..
+        } = event
+        {
+            for path in paths {
+                match self.watching.get_mut(&path) {
+                    Some((reloader, ref mut signals)) => {
+                        if !signals.is_empty() {
+                            // Reload this file
+                            reloader();
+
+                            signals.retain(|signal| match signal.upgrade() {
+                                Some(signal) => {
+                                    signal.store(true, Ordering::Release);
+                                    true
+                                }
+                                None => false,
+                            });
+                        }
+                        // If there is no one to signal stop watching this path
+                        if signals.is_empty() {
+                            // TODO: handle this result
+                            self.watcher.unwatch(&path);
+                            self.watching.remove(&path);
+                        }
+                    }
+                    None => {
+                        warn!("Watching {:#?} but there are no signals for this path. The path will be unwatched.", path);
+                        // TODO: handle this result
+                        self.watcher.unwatch(path);
+                    }
+                }
+            }
+        }
+    }
+    fn run(mut self) -> Sender<(PathBuf, Handler, Weak<AtomicBool>)> {
+        let (watch_tx, watch_rx) = unbounded();
+
+        thread::spawn(move || {
+            loop {
+                // TODO: handle errors
+                select! {
+                    recv(watch_rx) -> res => match res {
+                        Ok((path, handler, signal)) => self.watch(path, handler, signal),
+                        // Disconnected
+                        Err(_) => (),
+                    },
+                    recv(self.event_rx) -> res => match res {
+                        Ok(Ok(event)) => self.handle_event(event),
+                        // Notify Error
+                        Ok(Err(_)) => (),
+                        // Disconnected
+                        Err(_) => (),
+                    },
+                }
+            }
+        });
+
+        watch_tx
+    }
+}
+
+pub struct ReloadIndicator {
+    reloaded: Arc<AtomicBool>,
+    // Paths that have already been added
+    paths: Vec<PathBuf>,
+}
+impl ReloadIndicator {
+    pub fn new() -> Self {
+        Self {
+            reloaded: Arc::new(AtomicBool::new(false)),
+            paths: Vec::new(),
+        }
+    }
+    pub fn add<F>(&mut self, path: PathBuf, reloader: F)
+    where
+        F: 'static + Fn() + Send,
+    {
+        // Check to see if this was already added
+        if self.paths.iter().any(|p| *p == path) {
+            // Nothing else needs to be done
+            return;
+        } else {
+            self.paths.push(path.clone());
+        };
+
+        // TODO: handle result
+        WATCHER
+            .lock()
+            .unwrap()
+            .send((path, Box::new(reloader), Arc::downgrade(&self.reloaded)));
+    }
+    // Returns true if the watched file was changed
+    pub fn reloaded(&self) -> bool {
+        self.reloaded.swap(false, Ordering::Acquire)
+    }
+}

voxygen/src/render/renderer.rs

@@ -7,12 +7,14 @@ use super::{
     texture::Texture,
     Pipeline, RenderError,
 };
+use common::assets::{self, watch::ReloadIndicator};
 use gfx::{
     self,
     handle::Sampler,
     traits::{Device, Factory, FactoryExt},
 };
 use glsl_include::Context as IncludeContext;
+use log::error;
 use vek::*;
 
 /// Represents the format of the pre-processed color target.
@@ -64,6 +66,8 @@ pub struct Renderer {
     terrain_pipeline: GfxPipeline<terrain::pipe::Init<'static>>,
     ui_pipeline: GfxPipeline<ui::pipe::Init<'static>>,
     postprocess_pipeline: GfxPipeline<postprocess::pipe::Init<'static>>,
+
+    shader_reload_indicator: ReloadIndicator,
 }
 
 impl Renderer {
@@ -74,74 +78,10 @@ impl Renderer {
         win_color_view: WinColorView,
         win_depth_view: WinDepthView,
     ) -> Result<Self, RenderError> {
-        let globals = include_str!(concat!(
+        let mut shader_reload_indicator = ReloadIndicator::new();
-            env!("CARGO_MANIFEST_DIR"),
-            "/shaders/include/globals.glsl"
-        ));
-        let sky = include_str!(concat!(
-            env!("CARGO_MANIFEST_DIR"),
-            "/shaders/include/sky.glsl"
-        ));
-        let light = include_str!(concat!(
-            env!("CARGO_MANIFEST_DIR"),
-            "/shaders/include/light.glsl"
-        ));
 
-        let mut include_ctx = IncludeContext::new();
+        let (skybox_pipeline, figure_pipeline, terrain_pipeline, ui_pipeline, postprocess_pipeline) =
-        include_ctx.include("globals.glsl", globals);
+            create_pipelines(&mut factory, &mut shader_reload_indicator)?;
-        include_ctx.include("sky.glsl", sky);
-        include_ctx.include("light.glsl", light);
-
-        // Construct a pipeline for rendering skyboxes
-        let skybox_pipeline = create_pipeline(
-            &mut factory,
-            skybox::pipe::new(),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/skybox.vert")),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/skybox.frag")),
-            &include_ctx,
-        )?;
-
-        // Construct a pipeline for rendering figures
-        let figure_pipeline = create_pipeline(
-            &mut factory,
-            figure::pipe::new(),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/figure.vert")),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/figure.frag")),
-            &include_ctx,
-        )?;
-
-        // Construct a pipeline for rendering terrain
-        let terrain_pipeline = create_pipeline(
-            &mut factory,
-            terrain::pipe::new(),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/terrain.vert")),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/terrain.frag")),
-            &include_ctx,
-        )?;
-
-        // Construct a pipeline for rendering UI elements
-        let ui_pipeline = create_pipeline(
-            &mut factory,
-            ui::pipe::new(),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/ui.vert")),
-            include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/shaders/ui.frag")),
-            &include_ctx,
-        )?;
-
-        // Construct a pipeline for rendering our post-processing
-        let postprocess_pipeline = create_pipeline(
-            &mut factory,
-            postprocess::pipe::new(),
-            include_str!(concat!(
-                env!("CARGO_MANIFEST_DIR"),
-                "/shaders/postprocess.vert"
-            )),
-            include_str!(concat!(
-                env!("CARGO_MANIFEST_DIR"),
-                "/shaders/postprocess.frag"
-            )),
-            &include_ctx,
-        )?;
 
         let dims = win_color_view.get_dimensions();
         let (tgt_color_view, tgt_depth_view, tgt_color_res) =
@@ -168,6 +108,8 @@ impl Renderer {
             terrain_pipeline,
             ui_pipeline,
             postprocess_pipeline,
+
+            shader_reload_indicator,
         })
     }
 
@@ -242,6 +184,29 @@ impl Renderer {
     pub fn flush(&mut self) {
         self.encoder.flush(&mut self.device);
         self.device.cleanup();
+
+        // If the shaders files were changed attempt to recreate the shaders
+        if self.shader_reload_indicator.reloaded() {
+            match create_pipelines(&mut self.factory, &mut self.shader_reload_indicator) {
+                Ok((
+                    skybox_pipeline,
+                    figure_pipeline,
+                    terrain_pipline,
+                    ui_pipeline,
+                    postprocess_pipeline,
+                )) => {
+                    self.skybox_pipeline = skybox_pipeline;
+                    self.figure_pipeline = figure_pipeline;
+                    self.terrain_pipeline = terrain_pipline;
+                    self.ui_pipeline = ui_pipeline;
+                    self.postprocess_pipeline = postprocess_pipeline;
+                }
+                Err(e) => error!(
+                    "Could not recreate shaders from assets due to an error: {:#?}",
+                    e
+                ),
+            }
+        }
     }
 
     /// Create a new set of constants with the provided values.
@@ -488,6 +453,105 @@ struct GfxPipeline<P: gfx::pso::PipelineInit> {
     pso: gfx::pso::PipelineState<gfx_backend::Resources, P::Meta>,
 }
 
+/// Create new the pipelines used by the renderer.
+fn create_pipelines(
+    factory: &mut gfx_backend::Factory,
+    shader_reload_indicator: &mut ReloadIndicator,
+) -> Result<
+    (
+        GfxPipeline<skybox::pipe::Init<'static>>,
+        GfxPipeline<figure::pipe::Init<'static>>,
+        GfxPipeline<terrain::pipe::Init<'static>>,
+        GfxPipeline<ui::pipe::Init<'static>>,
+        GfxPipeline<postprocess::pipe::Init<'static>>,
+    ),
+    RenderError,
+> {
+    let globals =
+        assets::load_watched::<String>("voxygen.shaders.include.globals", shader_reload_indicator)
+            .unwrap();
+    let sky =
+        assets::load_watched::<String>("voxygen.shaders.include.sky", shader_reload_indicator)
+            .unwrap();
+    let light =
+        assets::load_watched::<String>("voxygen.shaders.include.light", shader_reload_indicator)
+            .unwrap();
+
+    let mut include_ctx = IncludeContext::new();
+    include_ctx.include("globals.glsl", &globals);
+    include_ctx.include("sky.glsl", &sky);
+    include_ctx.include("light.glsl", &light);
+
+    // Construct a pipeline for rendering skyboxes
+    let skybox_pipeline = create_pipeline(
+        factory,
+        skybox::pipe::new(),
+        &assets::load_watched::<String>("voxygen.shaders.skybox.vert", shader_reload_indicator)
+            .unwrap(),
+        &assets::load_watched::<String>("voxygen.shaders.skybox.frag", shader_reload_indicator)
+            .unwrap(),
+        &include_ctx,
+    )?;
+
+    // Construct a pipeline for rendering figures
+    let figure_pipeline = create_pipeline(
+        factory,
+        figure::pipe::new(),
+        &assets::load_watched::<String>("voxygen.shaders.figure.vert", shader_reload_indicator)
+            .unwrap(),
+        &assets::load_watched::<String>("voxygen.shaders.figure.frag", shader_reload_indicator)
+            .unwrap(),
+        &include_ctx,
+    )?;
+
+    // Construct a pipeline for rendering terrain
+    let terrain_pipeline = create_pipeline(
+        factory,
+        terrain::pipe::new(),
+        &assets::load_watched::<String>("voxygen.shaders.terrain.vert", shader_reload_indicator)
+            .unwrap(),
+        &assets::load_watched::<String>("voxygen.shaders.terrain.frag", shader_reload_indicator)
+            .unwrap(),
+        &include_ctx,
+    )?;
+
+    // Construct a pipeline for rendering UI elements
+    let ui_pipeline = create_pipeline(
+        factory,
+        ui::pipe::new(),
+        &assets::load_watched::<String>("voxygen.shaders.ui.vert", shader_reload_indicator)
+            .unwrap(),
+        &assets::load_watched::<String>("voxygen.shaders.ui.frag", shader_reload_indicator)
+            .unwrap(),
+        &include_ctx,
+    )?;
+
+    // Construct a pipeline for rendering our post-processing
+    let postprocess_pipeline = create_pipeline(
+        factory,
+        postprocess::pipe::new(),
+        &assets::load_watched::<String>(
+            "voxygen.shaders.postprocess.vert",
+            shader_reload_indicator,
+        )
+        .unwrap(),
+        &assets::load_watched::<String>(
+            "voxygen.shaders.postprocess.frag",
+            shader_reload_indicator,
+        )
+        .unwrap(),
+        &include_ctx,
+    )?;
+
+    Ok((
+        skybox_pipeline,
+        figure_pipeline,
+        terrain_pipeline,
+        ui_pipeline,
+        postprocess_pipeline,
+    ))
+}
+
 /// Create a new pipeline from the provided vertex shader and fragment shader.
 fn create_pipeline<'a, P: gfx::pso::PipelineInit>(
     factory: &mut gfx_backend::Factory,