Add ChonkCache that saves serialized chonks client-side and preloads their generation when requesting chunks.

Cargo.lock

@@ -5484,6 +5484,7 @@ version = "0.9.0"
 dependencies = [
  "async-channel",
  "authc",
+ "bincode",
  "byteorder",
  "clap",
  "futures-util",
@@ -5492,6 +5493,7 @@ dependencies = [
  "num 0.4.0",
  "rayon",
  "ron",
+ "rusqlite",
  "rustyline",
  "serde",
  "specs",

client/Cargo.toml

@@ -20,6 +20,7 @@ common-systems = { package = "veloren-common-systems", path = "../common/systems
 common-net = { package = "veloren-common-net", path = "../common/net" }
 network = { package = "veloren-network", path = "../network", features = ["compression"], default-features = false }
 
+bincode = "1.3.3"
 byteorder = "1.3.2"
 futures-util = "0.3.7"
 tokio = { version = "1", default-features = false, features = ["rt-multi-thread"] }
@@ -31,6 +32,7 @@ specs = { git = "https://github.com/amethyst/specs.git", rev = "5a9b71035007be0e
 vek = { version = "=0.14.1", features = ["serde"] }
 hashbrown = { version = "0.9", features = ["rayon", "serde", "nightly"] }
 authc = { git = "https://gitlab.com/veloren/auth.git", rev = "fb3dcbc4962b367253f8f2f92760ef44d2679c9a" }
+rusqlite = { version = "0.24.2", features = ["array", "vtab", "bundled", "trace"] }
 
 #TODO: put bot in a different crate
 #bot only

client/src/chonk_cache.rs

@@ -0,0 +1,86 @@
+use common_net::msg::SerializedTerrainChunk;
+use rusqlite::{Connection, ToSql};
+use tracing::warn;
+use vek::Vec2;
+
+pub struct ChonkCache {
+    connection: Option<Connection>,
+}
+
+impl ChonkCache {
+    /// Create a new ChonkCache using an in-memory database. If database
+    /// creation fails, returns a proxy that doesn't cache chunks instead of
+    /// propagating the failure.
+    pub fn new_in_memory() -> Self {
+        let mut ret = Self {
+            connection: Connection::open_in_memory().ok(),
+        };
+        if let Err(e) = ret.ensure_schema() {
+            warn!("Couldn't ensure schema for ChonkCache: {:?}", e);
+            ret.connection = None;
+        }
+        ret
+    }
+
+    #[rustfmt::skip]
+    fn ensure_schema(&self) -> rusqlite::Result<()> {
+        if let Some(connection) = &self.connection {
+            connection.execute_batch("
+                CREATE TABLE IF NOT EXISTS chonks (
+                    pos_x INTEGER NOT NULL,
+                    pos_y INTEGER NOT NULL,
+                    data BLOB NOT NULL
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS chonk_pos ON chonks(pos_x, pos_y);
+            ")
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Insert a chunk into the cache and return whether it already existed with
+    /// the same byte representation on success.
+    pub fn received_chonk(
+        &self,
+        key: Vec2<i32>,
+        chonk: &SerializedTerrainChunk,
+    ) -> Result<bool, Box<dyn std::error::Error>> {
+        if let Some(connection) = &self.connection {
+            let serialized = bincode::serialize(chonk)?;
+            let values = [&key.x as &dyn ToSql, &key.y, &serialized];
+
+            // TODO: ensure a canonical encoding for SerializedTerrainChunk/make caching
+            // more granular to get more hits
+            if connection
+                .prepare("SELECT NULL FROM chonks WHERE pos_x = ?1 AND pos_y = ?2 AND data = ?3")?
+                .query(&values)?
+                .next()?
+                .is_none()
+            {
+                connection.execute(
+                    "REPLACE INTO chonks (pos_x, pos_y, data) VALUES (?1, ?2, ?3)",
+                    &values,
+                )?;
+                Ok(false)
+            } else {
+                Ok(true)
+            }
+        } else {
+            Ok(false)
+        }
+    }
+
+    /// Check to see if there's a cached chonk at the specified index
+    pub fn get_cached_chonk(&self, key: Vec2<i32>) -> Option<SerializedTerrainChunk> {
+        self.connection.as_ref().and_then(|connection| {
+            connection
+                .query_row(
+                    "SELECT data FROM chonks WHERE pos_x = ?1 AND pos_y = ?2",
+                    &[key.x, key.y],
+                    |row| row.get(0),
+                )
+                .ok()
+                .and_then(|data: Vec<u8>| bincode::deserialize(&*data).ok())
+        })
+    }
+}

client/src/lib.rs

@@ -3,6 +3,7 @@
 #![feature(label_break_value, option_zip)]
 
 pub mod addr;
+pub mod chonk_cache;
 pub mod cmd;
 pub mod error;
 
@@ -18,6 +19,7 @@ pub use specs::{
 
 use crate::addr::ConnectionArgs;
 use byteorder::{ByteOrder, LittleEndian};
+use chonk_cache::ChonkCache;
 use common::{
     character::{CharacterId, CharacterItem},
     comp::{
@@ -192,6 +194,8 @@ pub struct Client {
 
     pending_chunks: HashMap<Vec2<i32>, Instant>,
     target_time_of_day: Option<TimeOfDay>,
+
+    chonk_cache: ChonkCache,
 }
 
 /// Holds data related to the current players characters, as well as some
@@ -699,6 +703,8 @@ impl Client {
 
             pending_chunks: HashMap::new(),
             target_time_of_day: None,
+
+            chonk_cache: ChonkCache::new_in_memory(),
         })
     }
 
@@ -1564,6 +1570,13 @@ impl Client {
                                         key: *key,
                                     })?;
                                     self.pending_chunks.insert(*key, Instant::now());
+                                    if let Some(chunk) = self
+                                        .chonk_cache
+                                        .get_cached_chonk(*key)
+                                        .and_then(|c| c.to_chunk())
+                                    {
+                                        self.state.insert_chunk(*key, Arc::new(chunk));
+                                    }
                                 } else {
                                     skip_mode = true;
                                 }
@@ -1961,8 +1974,19 @@ impl Client {
     fn handle_server_terrain_msg(&mut self, msg: ServerGeneral) -> Result<(), Error> {
         match msg {
             ServerGeneral::TerrainChunkUpdate { key, chunk } => {
-                if let Some(chunk) = chunk.ok().and_then(|c| c.to_chunk()) {
-                    self.state.insert_chunk(key, Arc::new(chunk));
+                if let Ok(wirechonk) = chunk {
+                    let should_insert = match self.chonk_cache.received_chonk(key, &wirechonk) {
+                        Ok(cache_hit) => !cache_hit,
+                        Err(e) => {
+                            warn!("Failed to insert chonk at {:?} into cache: {:?}", key, e);
+                            true
+                        },
+                    };
+                    if should_insert {
+                        if let Some(chunk) = wirechonk.to_chunk() {
+                            self.state.insert_chunk(key, Arc::new(chunk));
+                        }
+                    }
                 }
                 self.pending_chunks.remove(&key);
             },