Deserialize chunks off the main thread.

Cargo.lock

@@ -6470,6 +6470,7 @@ dependencies = [
  "bincode",
  "byteorder",
  "clap 3.1.8",
+ "crossbeam-channel",
  "hashbrown 0.12.0",
  "image",
  "num 0.4.0",

client/Cargo.toml

@@ -23,6 +23,7 @@ network = { package = "veloren-network", path = "../network", features = ["compr
 
 bincode = "1.3.2"
 byteorder = "1.3.2"
+crossbeam-channel = "0.5"
 tokio = { version = "1.14", default-features = false, features = ["rt-multi-thread"] }
 quinn = "0.8"
 image = { version = "0.24", default-features = false, features = ["png"] }

client/src/error.rs

@@ -1,4 +1,5 @@
 use authc::AuthClientError;
+use common_net::msg::DecodeError;
 pub use network::{InitProtocolError, NetworkConnectError, NetworkError};
 use network::{ParticipantError, StreamError};
 use specs::error::Error as SpecsError;
@@ -24,6 +25,7 @@ pub enum Error {
     //TODO: InvalidAlias,
     Other(String),
     SpecsErr(SpecsError),
+    CompressionError(DecodeError),
 }
 
 impl From<SpecsError> for Error {
@@ -46,6 +48,15 @@ impl From<bincode::Error> for Error {
     fn from(err: bincode::Error) -> Self { Self::StreamErr(StreamError::Deserialize(err)) }
 }
 
+impl From<DecodeError> for Error {
+    fn from(err: DecodeError) -> Self {
+        match err {
+            DecodeError::Deserialize(err) => Self::StreamErr(StreamError::Deserialize(err)),
+            _ => Self::CompressionError(err),
+        }
+    }
+}
+
 impl From<AuthClientError> for Error {
     fn from(err: AuthClientError) -> Self { Self::AuthClientError(err) }
 }

client/src/lib.rs

@@ -40,6 +40,7 @@ use common::{
     outcome::Outcome,
     recipe::{ComponentRecipeBook, RecipeBook},
     resources::{PlayerEntity, TimeOfDay},
+    slowjob::SlowJobPool,
     spiral::Spiral2d,
     terrain::{
         block::Block, map::MapConfig, neighbors, BiomeKind, SitesKind, SpriteKind, TerrainChunk,
@@ -66,6 +67,7 @@ use common_net::{
 use common_state::State;
 use common_systems::add_local_systems;
 use comp::BuffKind;
+use crossbeam_channel as mpsc;
 use hashbrown::{HashMap, HashSet};
 use image::DynamicImage;
 use network::{ConnectAddr, Network, Participant, Pid, Stream};
@@ -199,6 +201,19 @@ impl Default for WeatherLerp {
     }
 }
 
+/// Deserialized and decompressed terrain updates.
+enum TerrainUpdate {
+    Chunk {
+        key: Vec2<i32>,
+        chunk: Option<Arc<TerrainChunk>>,
+    },
+    LodZone {
+        key: Vec2<i32>,
+        zone: lod::Zone,
+    },
+    Block(HashMap<Vec3<i32>, Block>),
+}
+
 pub struct Client {
     registered: bool,
     presence: Option<PresenceKind>,
@@ -251,6 +266,8 @@ pub struct Client {
     // TODO: move into voxygen
     loaded_distance: f32,
 
+    terrain_tx: mpsc::Sender<Result<TerrainUpdate, Error>>,
+    terrain_rx: mpsc::Receiver<Result<TerrainUpdate, Error>>,
     pending_chunks: HashMap<Vec2<i32>, Instant>,
     target_time_of_day: Option<TimeOfDay>,
 }
@@ -263,6 +280,8 @@ pub struct CharacterList {
     pub loading: bool,
 }
 
+const TOTAL_PENDING_CHUNKS_LIMIT: usize = 1024;
+
 impl Client {
     pub async fn new(
         addr: ConnectionArgs,
@@ -355,6 +374,8 @@ impl Client {
                 let mut state = State::client();
                 // Client-only components
                 state.ecs_mut().register::<comp::Last<CharacterState>>();
+                state.ecs_mut().write_resource::<SlowJobPool>()
+                    .configure("TERRAIN_DESERIALIZING", |n| n / 2);
 
                 let entity = state.ecs_mut().apply_entity_package(entity_package);
                 *state.ecs_mut().write_resource() = time_of_day;
@@ -646,6 +667,8 @@ impl Client {
 
         debug!("Initial sync done");
 
+        let (terrain_tx, terrain_rx) = mpsc::bounded(TOTAL_PENDING_CHUNKS_LIMIT);
+
         Ok(Self {
             registered: false,
             presence: None,
@@ -707,6 +730,8 @@ impl Client {
             lod_distance: 4.0,
             loaded_distance: 0.0,
 
+            terrain_tx,
+            terrain_rx,
             pending_chunks: HashMap::new(),
             target_time_of_day: None,
         })
@@ -1827,7 +1852,6 @@ impl Client {
                     for key in keys.iter() {
                         if self.state.terrain().get_key(*key).is_none() {
                             if !skip_mode && !self.pending_chunks.contains_key(key) {
-                                const TOTAL_PENDING_CHUNKS_LIMIT: usize = 1024;
                                 const CURRENT_TICK_PENDING_CHUNKS_LIMIT: usize = 8 * 4;
                                 if self.pending_chunks.len() < TOTAL_PENDING_CHUNKS_LIMIT
                                     && current_tick_send_chunk_requests
@@ -2261,29 +2285,47 @@ impl Client {
         Ok(())
     }
 
-    fn handle_server_terrain_msg(&mut self, msg: ServerGeneral) -> Result<(), Error> {
+    fn handle_server_terrain_msg(msg: ServerGeneral) -> Result<TerrainUpdate, Error> {
         prof_span!("handle_server_terrain_mgs");
         match msg {
             ServerGeneral::TerrainChunkUpdate { key, chunk } => {
-                if let Some(chunk) = chunk.ok().and_then(|c| c.to_chunk()) {
+                let chunk = chunk
-                    self.state.insert_chunk(key, Arc::new(chunk));
+                    .ok()
+                    .map(|c| c.to_chunk())
+                    .transpose()?
+                    .map(Arc::new);
+                return Ok(TerrainUpdate::Chunk { key, chunk });
+            },
+            ServerGeneral::LodZoneUpdate { key, zone } => {
+                return Ok(TerrainUpdate::LodZone { key, zone });
+            },
+            ServerGeneral::TerrainBlockUpdates(blocks) => {
+                let blocks = blocks.decompress()?;
+                return Ok(TerrainUpdate::Block(blocks));
+            },
+            _ => {},
+        }
+        return Err(Error::Other("Invalid terrain message".into()));
+    }
+
+    fn handle_terrain_msg(&mut self, msg: TerrainUpdate) {
+        match msg {
+            TerrainUpdate::Chunk { key, chunk } => {
+                if let Some(chunk) = chunk {
+                    self.state.insert_chunk(key, chunk);
                 }
                 self.pending_chunks.remove(&key);
             },
-            ServerGeneral::LodZoneUpdate { key, zone } => {
+            TerrainUpdate::LodZone { key, zone } => {
                 self.lod_zones.insert(key, zone);
                 self.lod_last_requested = None;
             },
-            ServerGeneral::TerrainBlockUpdates(blocks) => {
+            TerrainUpdate::Block(blocks) => {
-                if let Some(mut blocks) = blocks.decompress() {
+                blocks.into_iter().for_each(|(pos, block)| {
-                    blocks.drain().for_each(|(pos, block)| {
+                    self.state.set_block(pos, block);
-                        self.state.set_block(pos, block);
+                });
-                    });
+            }
-                }
-            },
-            _ => unreachable!("Not a terrain message"),
         }
-        Ok(())
     }
 
     fn handle_server_character_screen_msg(
@@ -2375,15 +2417,28 @@ impl Client {
             }
             while let Some(msg) = self.terrain_stream.try_recv_raw()? {
                 cnt += 1;
-                let msg = msg.decompress()?;
+                let terrain_tx = self.terrain_tx.clone();
-                let msg = bincode::deserialize(&msg)?;
+                self
+                    .state
+                    .slow_job_pool()
+                    .spawn("TERRAIN_DESERIALIZING", move || {
+                        let handle_msg = || {
+                            let msg = msg.decompress()?;
+                            let msg = bincode::deserialize(&msg)?;
+                            Self::handle_server_terrain_msg(msg)
+                        };
+                        terrain_tx.send(handle_msg());
+                    });
+            }
+            while let Ok(msg) = self.terrain_rx.try_recv() {
+                let msg = msg?;
                 #[cfg(feature = "tracy")]
                 {
-                    if let ServerGeneral::TerrainChunkUpdate { chunk, .. } = &msg {
+                    if let TerrainUpdate::Chunk { chunk, .. } = &msg {
                         terrain_cnt += chunk.as_ref().map(|x| x.approx_len()).unwrap_or(0);
                     }
                 }
-                self.handle_server_terrain_msg(msg)?;
+                self.handle_terrain_msg(msg);
             }
 
             if cnt_start == cnt {

common/net/src/msg/compression.rs

@@ -4,19 +4,55 @@ use common::{
     volumes::vol_grid_2d::VolGrid2d,
 };
 use hashbrown::HashMap;
-use image::{ImageBuffer, ImageDecoder, ImageEncoder, Pixel};
+use image::{ImageBuffer, ImageDecoder, ImageEncoder, ImageError, Pixel};
 use num_traits::cast::FromPrimitive;
 use serde::{Deserialize, Serialize};
 use std::{
     borrow::Cow,
     fmt::Debug,
-    io::{Read, Write},
+    io::{Error as IoError, Read, Write},
     marker::PhantomData,
 };
 use serde_with::{serde_as, Bytes};
 use tracing::warn;
 use vek::*;
 
+/// Errors that can occur during decoding.
+#[derive(Debug)]
+pub enum DecodeError {
+    Deserialize(bincode::Error),
+    Image(ImageError),
+    Io(IoError)
+}
+
+impl ToString for DecodeError {
+    fn to_string(&self) -> String {
+        match self {
+            Self::Deserialize(error) => error.to_string(),
+            Self::Image(error) => error.to_string(),
+            Self::Io(error) => error.to_string(),
+        }
+    }
+}
+
+impl From<ImageError> for DecodeError {
+    fn from(error: ImageError) -> Self {
+        Self::Image(error)
+    }
+}
+
+impl From<IoError> for DecodeError {
+    fn from(error: IoError) -> Self {
+        Self::Io(error)
+    }
+}
+
+impl From<bincode::Error> for DecodeError {
+    fn from(error: bincode::Error) -> Self {
+        Self::Deserialize(error)
+    }
+}
+
 /// Wrapper for compressed, serialized data (for stuff that doesn't use the
 /// default lz4 compression)
 #[serde_as]
@@ -59,15 +95,14 @@ impl<'a, T: Serialize> CompressedData<'a, T> {
 }
 
 impl<T: for<'a> Deserialize<'a>> CompressedData<'_, T> {
-    pub fn decompress(&self) -> Option<T> {
+    pub fn decompress(&self) -> Result<T, DecodeError> {
         if self.compressed {
             let mut uncompressed = Vec::with_capacity(self.data.len());
             flate2::read::DeflateDecoder::new(&*self.data)
-                .read_to_end(&mut uncompressed)
+                .read_to_end(&mut uncompressed)?;
-                .ok()?;
+            Ok(bincode::deserialize(&*uncompressed)?)
-            bincode::deserialize(&*uncompressed).ok()
         } else {
-            bincode::deserialize(&*self.data).ok()
+            Ok(bincode::deserialize(&*self.data)?)
         }
     }
 }
@@ -129,6 +164,8 @@ impl PackingFormula for GridLtrPacking {
 pub trait VoxelImageEncoding {
     type Workspace;
     type Output;
+    type EncodeError;
+
     fn create(width: u32, height: u32) -> Self::Workspace;
     fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>);
     fn put_sprite(
@@ -140,25 +177,31 @@ pub trait VoxelImageEncoding {
         sprite: SpriteKind,
         ori: Option<u8>,
     );
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output>;
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError>;
 }
 
 pub trait VoxelImageDecoding: VoxelImageEncoding {
-    fn start(ws: &Self::Output) -> Option<Self::Workspace>;
+    fn start(ws: &Self::Output) -> Result<Self::Workspace, DecodeError>;
     fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block;
 }
 
 pub fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u8>>(
     decoder: I,
-) -> Option<ImageBuffer<P, Vec<u8>>> {
+) -> Result<ImageBuffer<P, Vec<u8>>, ImageError> {
     let (w, h) = decoder.dimensions();
     let mut buf = vec![0; decoder.total_bytes() as usize];
-    decoder.read_image(&mut buf).ok()?;
+    decoder.read_image(&mut buf)?;
-    ImageBuffer::from_raw(w, h, buf)
+    Ok(ImageBuffer::from_raw(w, h, buf).ok_or_else(|| {
+        IoError::new(
+            std::io::ErrorKind::WriteZero,
+            "Could not write full image to buffer; maybe out of memory?"
+        )
+    })?)
 }
 
 impl<'a, VIE: VoxelImageEncoding> VoxelImageEncoding for &'a VIE {
     type Output = VIE::Output;
+    type EncodeError = VIE::EncodeError;
     type Workspace = VIE::Workspace;
 
     fn create(width: u32, height: u32) -> Self::Workspace { VIE::create(width, height) }
@@ -179,11 +222,11 @@ impl<'a, VIE: VoxelImageEncoding> VoxelImageEncoding for &'a VIE {
         (*self).put_sprite(ws, x, y, kind, sprite, ori)
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> { VIE::finish(ws) }
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, VIE::EncodeError> { VIE::finish(ws) }
 }
 
 impl<'a, VIE: VoxelImageDecoding> VoxelImageDecoding for &'a VIE {
-    fn start(ws: &Self::Output) -> Option<Self::Workspace> { VIE::start(ws) }
+    fn start(ws: &Self::Output) -> Result<Self::Workspace, DecodeError> { VIE::start(ws) }
 
     fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block {
         VIE::get_block(ws, x, y, is_border)
@@ -195,6 +238,7 @@ pub struct QuadPngEncoding<'a, const RESOLUTION_DIVIDER: u32>(pub PhantomData<&'
 
 impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
     type Output = CompressedData<'a, (Vec<u8>, [usize; 3])>;
+    type EncodeError = ImageError;
     type Workspace = (
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
@@ -232,20 +276,19 @@ impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
         ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let mut indices = [0; 3];
-        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
             let png = image::codecs::png::PngEncoder::new_with_quality(
                 &mut buf,
                 CompressionType::Rle,
                 FilterType::Up,
             );
-            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
-                .ok()?;
             indices[i] = buf.len();
-            Some(())
+            Ok(())
         };
         f(&ws.0, 0)?;
         f(&ws.1, 1)?;
@@ -262,11 +305,10 @@ impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
                 ws.3.width(),
                 ws.3.height(),
                 image::ColorType::Rgb8,
-            )
+            )?;
-            .ok()?;
         }
 
-        Some(CompressedData::compress(&(buf, indices), 4))
+        Ok(CompressedData::compress(&(buf, indices), 4))
     }
 }
 
@@ -323,7 +365,7 @@ const fn gen_lanczos_lookup<const N: u32, const R: u32>(
 }
 
 impl<const N: u32> VoxelImageDecoding for QuadPngEncoding<'_, N> {
-    fn start(data: &Self::Output) -> Option<Self::Workspace> {
+    fn start(data: &Self::Output) -> Result<Self::Workspace, DecodeError> {
         use image::codecs::png::PngDecoder;
         let (quad, indices) = data.decompress()?;
         let ranges: [_; 4] = [
@@ -332,11 +374,11 @@ impl<const N: u32> VoxelImageDecoding for QuadPngEncoding<'_, N> {
             indices[1]..indices[2],
             indices[2]..quad.len(),
         ];
-        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
+        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
-        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
+        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
-        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
+        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
-        let d = image_from_bytes(PngDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
+        let d = image_from_bytes(PngDecoder::new(&quad[ranges[3].clone()])?)?;
-        Some((a, b, c, d))
+        Ok((a, b, c, d))
     }
 
     fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block {
@@ -469,6 +511,7 @@ pub struct TriPngEncoding<'a, const AVERAGE_PALETTE: bool>(pub PhantomData<&'a (
 
 impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a, AVERAGE_PALETTE> {
     type Output = CompressedData<'a, (Vec<u8>, Vec<Rgb<u8>>, [usize; 3])>;
+    type EncodeError = ImageError;
     type Workspace = (
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
@@ -508,20 +551,19 @@ impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a,
         ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let mut indices = [0; 3];
-        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
             let png = image::codecs::png::PngEncoder::new_with_quality(
                 &mut buf,
                 CompressionType::Rle,
                 FilterType::Up,
             );
-            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
-                .ok()?;
             indices[i] = buf.len();
-            Some(())
+            Ok(())
         };
         f(&ws.0, 0)?;
         f(&ws.1, 1)?;
@@ -550,12 +592,12 @@ impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a,
             Vec::new()
         };
 
-        Some(CompressedData::compress(&(buf, palette, indices), 4))
+        Ok(CompressedData::compress(&(buf, palette, indices), 4))
     }
 }
 
 impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVERAGE_PALETTE> {
-    fn start(data: &Self::Output) -> Option<Self::Workspace> {
+    fn start(data: &Self::Output) -> Result<Self::Workspace, DecodeError> {
         use image::codecs::png::PngDecoder;
         let (quad, palette, indices) = data.decompress()?;
         let ranges: [_; 3] = [
@@ -563,9 +605,9 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVER
             indices[0]..indices[1],
             indices[1]..indices[2],
         ];
-        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
+        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
-        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
+        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
-        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
+        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
         let mut d: HashMap<_, HashMap<_, _>> = HashMap::new();
         if AVERAGE_PALETTE {
             for i in 0..=255 {
@@ -578,7 +620,7 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVER
             }
         }
 
-        Some((a, b, c, d))
+        Ok((a, b, c, d))
     }
 
     fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
@@ -681,7 +723,7 @@ pub fn image_terrain_chonk<S: RectVolSize, Storage: core::ops::DerefMut<Target=V
     vie: &VIE,
     packing: P,
     chonk: &Chonk<Block, Storage, S, M>,
-) -> Option<VIE::Output> {
+) -> Result<VIE::Output, VIE::EncodeError> {
     image_terrain(
         vie,
         packing,
@@ -701,7 +743,7 @@ pub fn image_terrain_volgrid<
     vie: &VIE,
     packing: P,
     volgrid: &VolGrid2d<Chonk<Block, Storage, S, M>>,
-) -> Option<VIE::Output> {
+) -> Result<VIE::Output, VIE::EncodeError> {
     let mut lo = Vec3::broadcast(i32::MAX);
     let mut hi = Vec3::broadcast(i32::MIN);
     for (pos, chonk) in volgrid.iter() {
@@ -727,7 +769,7 @@ pub fn image_terrain<
     vol: &V,
     lo: Vec3<u32>,
     hi: Vec3<u32>,
-) -> Option<VIE::Output> {
+) -> Result<VIE::Output, VIE::EncodeError> {
     let dims = Vec3::new(
         hi.x.wrapping_sub(lo.x),
         hi.y.wrapping_sub(lo.y),
@@ -782,7 +824,7 @@ pub fn write_image_terrain<
     data: &VIE::Output,
     lo: Vec3<u32>,
     hi: Vec3<u32>,
-) -> Option<()> {
+) -> Result<(), DecodeError> {
     let ws = VIE::start(data)?;
     let dims = Vec3::new(
         hi.x.wrapping_sub(lo.x),
@@ -805,7 +847,7 @@ pub fn write_image_terrain<
             }
         }
     }
-    Some(())
+    Ok(())
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -824,16 +866,14 @@ pub struct WireChonk<VIE: VoxelImageEncoding, P: PackingFormula, M: Clone, S: Re
 impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone, S: RectVolSize>
     WireChonk<VIE, P, M, S>
 {
-    pub fn from_chonk<Storage: core::ops::DerefMut<Target=Vec<Block>>>(vie: VIE, packing: P, chonk: &Chonk<Block, Storage, S, M>) -> Option<Self> {
+    pub fn from_chonk<Storage: core::ops::DerefMut<Target=Vec<Block>>>(vie: VIE, packing: P, chonk: &Chonk<Block, Storage, S, M>) -> Result<Self, VIE::EncodeError> {
         let data = image_terrain_chonk(&vie, packing, chonk)?;
-        Some(Self {
+        Ok(Self {
             zmin: chonk.get_min_z(),
             zmax: chonk.get_max_z(),
             data,
-            below: *chonk
+            below: *chonk.below(),
-                .get(Vec3::new(0, 0, chonk.get_min_z().saturating_sub(1)))
+            above: *chonk.above(),
-                .ok()?,
-            above: *chonk.get(Vec3::new(0, 0, chonk.get_max_z() + 1)).ok()?,
             meta: chonk.meta().clone(),
             vie,
             packing,
@@ -841,7 +881,7 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
         })
     }
 
-    pub fn to_chonk<Storage: Clone + core::ops::DerefMut<Target=Vec<Block>> + From<Vec<Block>>>(&self) -> Option<Chonk<Block, Storage, S, M>> {
+    pub fn to_chonk<Storage: Clone + core::ops::DerefMut<Target=Vec<Block>> + From<Vec<Block>>>(&self) -> Result<Chonk<Block, Storage, S, M>, DecodeError> {
         let mut chonk = Chonk::new(self.zmin, self.below, self.above, self.meta.clone());
         write_image_terrain(
             &self.vie,
@@ -851,6 +891,6 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
             Vec3::new(0, 0, self.zmin as u32),
             Vec3::new(S::RECT_SIZE.x, S::RECT_SIZE.y, self.zmax as u32),
         )?;
-        Some(chonk)
+        Ok(chonk)
     }
 }

common/net/src/msg/mod.rs

@@ -8,7 +8,7 @@ pub mod world_msg;
 pub use self::{
     client::{ClientGeneral, ClientMsg, ClientRegister, ClientType},
     compression::{
-        CompressedData, GridLtrPacking, PackingFormula, QuadPngEncoding, TriPngEncoding,
+        CompressedData, DecodeError, GridLtrPacking, PackingFormula, QuadPngEncoding, TriPngEncoding,
         VoxelImageEncoding, WidePacking, WireChonk,
     },
     ecs_packet::EcsCompPacket,

common/net/src/msg/server.rs

@@ -1,5 +1,5 @@
 use super::{
-    world_msg::EconomyInfo, ClientType, CompressedData, EcsCompPacket, PingMsg, QuadPngEncoding,
+    world_msg::EconomyInfo, ClientType, CompressedData, DecodeError, EcsCompPacket, PingMsg, QuadPngEncoding,
     TriPngEncoding, WidePacking, WireChonk,
 };
 use crate::sync;
@@ -104,24 +104,26 @@ impl SerializedTerrainChunk<'_> {
     }
 
     pub fn quadpng(chunk: &TerrainChunk) -> Self {
-        if let Some(wc) = WireChonk::from_chonk(QuadPngEncoding(PhantomData), WidePacking(), chunk) {
+        match WireChonk::from_chonk(QuadPngEncoding(PhantomData), WidePacking(), chunk) {
-            Self::QuadPng(wc)
+            Ok(wc) => Self::QuadPng(wc),
-        } else {
+            Err(err) => {
-            warn!("Image encoding failure occurred, falling back to deflate");
+                warn!("Image encoding failure occurred, falling back to deflate: {}", err);
-            Self::deflate(chunk)
+                Self::deflate(chunk)
+            }
         }
     }
 
     pub fn tripng(chunk: &TerrainChunk) -> Self {
-        if let Some(wc) = WireChonk::from_chonk(TriPngEncoding(PhantomData), WidePacking(), chunk) {
+        match WireChonk::from_chonk(TriPngEncoding(PhantomData), WidePacking(), chunk) {
-            Self::TriPng(wc)
+            Ok(wc) => Self::TriPng(wc),
-        } else {
+            Err(err) => {
-            warn!("Image encoding failure occurred, falling back to deflate");
+                warn!("Image encoding failure occurred, falling back to deflate: {}", err);
-            Self::deflate(chunk)
+                Self::deflate(chunk)
+            },
         }
     }
 
-    pub fn to_chunk(&self) -> Option<TerrainChunk> {
+    pub fn to_chunk(&self) -> Result<TerrainChunk, DecodeError> {
         match self {
             Self::DeflatedChonk(chonk) => chonk.decompress(),
             Self::QuadPng(wc) => wc.to_chonk(),

common/src/terrain/chonk.rs

@@ -84,6 +84,10 @@ impl<V, Storage: core::ops::DerefMut<Target=Vec<V>>, S: RectVolSize, M: Clone> C
         }
     }
 
+    pub fn below(&self) -> &V { &self.below }
+
+    pub fn above(&self) -> &V { &self.above }
+
     pub fn meta(&self) -> &M { &self.meta }
 
     #[inline]

voxygen/src/menu/main/mod.rs

@@ -432,6 +432,7 @@ fn get_client_msg_error(
                 localization.get("main.login.authentication_error"),
                 e
             ),
+            Error::CompressionError(e) => net_error(e.to_string(), mismatched_server_info),
             Error::Kicked(e) => e,
             Error::TooManyPlayers => localization.get("main.login.server_full").into(),
             Error::AuthServerNotTrusted => {

world/examples/chunk_compression_benchmarks.rs

@@ -9,13 +9,13 @@ use common::{
     },
 };
 use common_net::msg::compression::{
-    image_from_bytes, image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking,
+    image_from_bytes, image_terrain_chonk, image_terrain_volgrid, CompressedData, DecodeError, GridLtrPacking,
     PackingFormula, QuadPngEncoding, TriPngEncoding, VoxelImageDecoding, VoxelImageEncoding,
     WidePacking,
 };
 use core::marker::PhantomData;
 use hashbrown::HashMap;
-use image::{ImageBuffer, ImageEncoder};
+use image::{ImageBuffer, ImageEncoder, ImageError};
 use num_traits::cast::FromPrimitive;
 use rayon::ThreadPoolBuilder;
 use serde::{Deserialize, Serialize};
@@ -166,6 +166,7 @@ pub struct PngEncoding;
 
 impl VoxelImageEncoding for PngEncoding {
     type Output = Vec<u8>;
+    type EncodeError = ImageError;
     type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
 
     fn create(width: u32, height: u32) -> Self::Workspace {
@@ -193,7 +194,7 @@ impl VoxelImageEncoding for PngEncoding {
         );
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         use image::codecs::png::{CompressionType, FilterType};
         let mut buf = Vec::new();
         let png = image::codecs::png::PngEncoder::new_with_quality(
@@ -206,9 +207,8 @@ impl VoxelImageEncoding for PngEncoding {
             ws.width(),
             ws.height(),
             image::ColorType::Rgba8,
-        )
+        )?;
-        .ok()?;
+        Ok(buf)
-        Some(buf)
     }
 }
 
@@ -217,6 +217,7 @@ pub struct JpegEncoding;
 
 impl VoxelImageEncoding for JpegEncoding {
     type Output = Vec<u8>;
+    type EncodeError = ImageError;
     type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
 
     fn create(width: u32, height: u32) -> Self::Workspace {
@@ -240,11 +241,11 @@ impl VoxelImageEncoding for JpegEncoding {
         ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
-        jpeg.encode_image(ws).ok()?;
+        jpeg.encode_image(ws)?;
-        Some(buf)
+        Ok(buf)
     }
 }
 
@@ -253,6 +254,7 @@ pub struct MixedEncoding;
 
 impl VoxelImageEncoding for MixedEncoding {
     type Output = (Vec<u8>, [usize; 3]);
+    type EncodeError = ImageError;
     type Workspace = (
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
@@ -291,33 +293,32 @@ impl VoxelImageEncoding for MixedEncoding {
         ws.3.put_pixel(x, y, image::Rgb([0; 3]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let mut indices = [0; 3];
-        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
             let png = image::codecs::png::PngEncoder::new_with_quality(
                 &mut buf,
                 CompressionType::Rle,
                 FilterType::Up,
             );
-            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
-                .ok()?;
             indices[i] = buf.len();
-            Some(())
+            Ok(())
         };
         f(&ws.0, 0)?;
         f(&ws.1, 1)?;
         f(&ws.2, 2)?;
 
         let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 10);
-        jpeg.encode_image(&ws.3).ok()?;
+        jpeg.encode_image(&ws.3)?;
-        Some((buf, indices))
+        Ok((buf, indices))
     }
 }
 
 impl VoxelImageDecoding for MixedEncoding {
-    fn start((quad, indices): &Self::Output) -> Option<Self::Workspace> {
+    fn start((quad, indices): &Self::Output) -> Result<Self::Workspace, DecodeError> {
         use image::codecs::{jpeg::JpegDecoder, png::PngDecoder};
         let ranges: [_; 4] = [
             0..indices[0],
@@ -325,11 +326,11 @@ impl VoxelImageDecoding for MixedEncoding {
             indices[1]..indices[2],
             indices[2]..quad.len(),
         ];
-        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
+        let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
-        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
+        let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
-        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
+        let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
-        let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
+        let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()])?)?;
-        Some((a, b, c, d))
+        Ok((a, b, c, d))
     }
 
     fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
@@ -366,6 +367,7 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
         usize,
         CompressedData<'static, HashMap<Vec2<u32>, (SpriteKind, u8)>>,
     );
+    type EncodeError = ImageError;
     type Workspace = (
         image::ImageBuffer<image::Luma<u8>, Vec<u8>>,
         image::ImageBuffer<image::Rgb<u8>, Vec<u8>>,
@@ -399,7 +401,7 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
         ws.2.insert(Vec2::new(x, y), (sprite, ori.unwrap_or(0)));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let png = image::codecs::png::PngEncoder::new_with_quality(
@@ -412,12 +414,11 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
             ws.0.width(),
             ws.0.height(),
             image::ColorType::L8,
-        )
+        )?;
-        .ok()?;
         let index = buf.len();
         let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
-        jpeg.encode_image(&ws.1).ok()?;
+        jpeg.encode_image(&ws.1)?;
-        Some((buf, index, CompressedData::compress(&ws.2, 4)))
+        Ok((buf, index, CompressedData::compress(&ws.2, 4)))
     }
 }
 
@@ -426,6 +427,7 @@ pub struct MixedEncodingDenseSprites;
 
 impl VoxelImageEncoding for MixedEncodingDenseSprites {
     type Output = (Vec<u8>, [usize; 3]);
+    type EncodeError = ImageError;
     type Workspace = (
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
         Vec<u8>,
@@ -462,20 +464,19 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
         ws.3.put_pixel(x, y, image::Rgb([0; 3]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let mut indices = [0; 3];
-        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
             let png = image::codecs::png::PngEncoder::new_with_quality(
                 &mut buf,
                 CompressionType::Fast,
                 FilterType::Up,
             );
-            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
-                .ok()?;
             indices[i] = buf.len();
-            Some(())
+            Ok(())
         };
         f(&ws.0, 0)?;
         let mut g = |x: &[u8], i| {
@@ -487,8 +488,8 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
         g(&ws.2, 2);
 
         let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
-        jpeg.encode_image(&ws.3).ok()?;
+        jpeg.encode_image(&ws.3)?;
-        Some((buf, indices))
+        Ok((buf, indices))
     }
 }
 
@@ -592,6 +593,7 @@ pub struct PaletteEncoding<'a, NN: NearestNeighbor, const N: u32>(&'a HashMap<Bl
 
 impl<'a, NN: NearestNeighbor, const N: u32> VoxelImageEncoding for PaletteEncoding<'a, NN, N> {
     type Output = CompressedData<'a, (Vec<u8>, [usize; 4])>;
+    type EncodeError = ImageError;
     type Workspace = (
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
         ImageBuffer<image::Luma<u8>, Vec<u8>>,
@@ -628,27 +630,26 @@ impl<'a, NN: NearestNeighbor, const N: u32> VoxelImageEncoding for PaletteEncodi
         ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
     }
 
-    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+    fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
         let mut buf = Vec::new();
         use image::codecs::png::{CompressionType, FilterType};
         let mut indices = [0; 4];
-        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
             let png = image::codecs::png::PngEncoder::new_with_quality(
                 &mut buf,
                 CompressionType::Rle,
                 FilterType::Up,
             );
-            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+            png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
-                .ok()?;
             indices[i] = buf.len();
-            Some(())
+            Ok(())
         };
         f(&ws.0, 0)?;
         f(&ws.1, 1)?;
         f(&ws.2, 2)?;
         f(&ws.3, 3)?;
 
-        Some(CompressedData::compress(&(buf, indices), 1))
+        Ok(CompressedData::compress(&(buf, indices), 1))
     }
 }
 

world/src/civ/mod.rs

@@ -1347,7 +1347,7 @@ impl SiteKind {
                 },
                 SiteKind::Citadel => (-0.3..0.7).contains(&chunk.temp) && chunk.tree_density < 0.4,
                 SiteKind::GiantTree/* | SiteKind::Tree*/ => {
-                    chunk.tree_density > 0.4 && (-0.3..0.4).contains(&chunk.temp)
+                    chunk.tree_density > 0.4/* && (-0.3..0.4).contains(&chunk.temp) */
                 },
                 SiteKind::CliffTown => {
                     (-0.6..0.4).contains(&chunk.temp)