Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deserialize chunks off the main thread.

Browse Source
This commit is contained in:
Joshua Yanovski 2022-07-07 21:06:56 -07:00
parent 6eb26ba6d8
commit 3ba53b3e4c
11 changed files with 240 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -6470,6 +6470,7 @@ dependencies = [
"bincode",
"byteorder",
"clap 3.1.8",
"crossbeam-channel",
"hashbrown 0.12.0",
"image",
"num 0.4.0",

1
client/Cargo.toml
View File

@ -23,6 +23,7 @@ network = { package = "veloren-network", path = "../network", features = ["compr
bincode = "1.3.2"
byteorder = "1.3.2"
crossbeam-channel = "0.5"
tokio = { version = "1.14", default-features = false, features = ["rt-multi-thread"] }
quinn = "0.8"
image = { version = "0.24", default-features = false, features = ["png"] }

11
client/src/error.rs
View File

@ -1,4 +1,5 @@
use authc::AuthClientError;
use common_net::msg::DecodeError;
pub use network::{InitProtocolError, NetworkConnectError, NetworkError};
use network::{ParticipantError, StreamError};
use specs::error::Error as SpecsError;
@ -24,6 +25,7 @@ pub enum Error {
//TODO: InvalidAlias,
Other(String),
SpecsErr(SpecsError),
CompressionError(DecodeError),
}
impl From<SpecsError> for Error {
@ -46,6 +48,15 @@ impl From<bincode::Error> for Error {
fn from(err: bincode::Error) -> Self { Self::StreamErr(StreamError::Deserialize(err)) }
}
impl From<DecodeError> for Error {
fn from(err: DecodeError) -> Self {
match err {
DecodeError::Deserialize(err) => Self::StreamErr(StreamError::Deserialize(err)),
_ => Self::CompressionError(err),
}
}
}
impl From<AuthClientError> for Error {
fn from(err: AuthClientError) -> Self { Self::AuthClientError(err) }
}

81
client/src/lib.rs
View File

@ -40,6 +40,7 @@ use common::{
outcome::Outcome,
recipe::{ComponentRecipeBook, RecipeBook},
resources::{PlayerEntity, TimeOfDay},
slowjob::SlowJobPool,
spiral::Spiral2d,
terrain::{
block::Block, map::MapConfig, neighbors, BiomeKind, SitesKind, SpriteKind, TerrainChunk,
@ -66,6 +67,7 @@ use common_net::{
use common_state::State;
use common_systems::add_local_systems;
use comp::BuffKind;
use crossbeam_channel as mpsc;
use hashbrown::{HashMap, HashSet};
use image::DynamicImage;
use network::{ConnectAddr, Network, Participant, Pid, Stream};
@ -199,6 +201,19 @@ impl Default for WeatherLerp {
}
}
/// Deserialized and decompressed terrain updates.
enum TerrainUpdate {
Chunk {
key: Vec2<i32>,
chunk: Option<Arc<TerrainChunk>>,
},
LodZone {
key: Vec2<i32>,
zone: lod::Zone,
},
Block(HashMap<Vec3<i32>, Block>),
}
pub struct Client {
registered: bool,
presence: Option<PresenceKind>,
@ -251,6 +266,8 @@ pub struct Client {
// TODO: move into voxygen
loaded_distance: f32,
terrain_tx: mpsc::Sender<Result<TerrainUpdate, Error>>,
terrain_rx: mpsc::Receiver<Result<TerrainUpdate, Error>>,
pending_chunks: HashMap<Vec2<i32>, Instant>,
target_time_of_day: Option<TimeOfDay>,
}
@ -263,6 +280,8 @@ pub struct CharacterList {
pub loading: bool,
}
const TOTAL_PENDING_CHUNKS_LIMIT: usize = 1024;
impl Client {
pub async fn new(
addr: ConnectionArgs,
@ -355,6 +374,8 @@ impl Client {
let mut state = State::client();
// Client-only components
state.ecs_mut().register::<comp::Last<CharacterState>>();
state.ecs_mut().write_resource::<SlowJobPool>()
.configure("TERRAIN_DESERIALIZING", |n| n / 2);
let entity = state.ecs_mut().apply_entity_package(entity_package);
*state.ecs_mut().write_resource() = time_of_day;
@ -646,6 +667,8 @@ impl Client {
debug!("Initial sync done");
let (terrain_tx, terrain_rx) = mpsc::bounded(TOTAL_PENDING_CHUNKS_LIMIT);
Ok(Self {
registered: false,
presence: None,
@ -707,6 +730,8 @@ impl Client {
lod_distance: 4.0,
loaded_distance: 0.0,
terrain_tx,
terrain_rx,
pending_chunks: HashMap::new(),
target_time_of_day: None,
})
@ -1827,7 +1852,6 @@ impl Client {
for key in keys.iter() {
if self.state.terrain().get_key(*key).is_none() {
if !skip_mode && !self.pending_chunks.contains_key(key) {
const TOTAL_PENDING_CHUNKS_LIMIT: usize = 1024;
const CURRENT_TICK_PENDING_CHUNKS_LIMIT: usize = 8 * 4;
if self.pending_chunks.len() < TOTAL_PENDING_CHUNKS_LIMIT
&& current_tick_send_chunk_requests
@ -2261,29 +2285,47 @@ impl Client {
Ok(())
}
fn handle_server_terrain_msg(&mut self, msg: ServerGeneral) -> Result<(), Error> {
fn handle_server_terrain_msg(msg: ServerGeneral) -> Result<TerrainUpdate, Error> {
prof_span!("handle_server_terrain_mgs");
match msg {
ServerGeneral::TerrainChunkUpdate { key, chunk } => {
if let Some(chunk) = chunk.ok().and_then(|c| c.to_chunk()) {
self.state.insert_chunk(key, Arc::new(chunk));
let chunk = chunk
.ok()
.map(|c| c.to_chunk())
.transpose()?
.map(Arc::new);
return Ok(TerrainUpdate::Chunk { key, chunk });
},
ServerGeneral::LodZoneUpdate { key, zone } => {
return Ok(TerrainUpdate::LodZone { key, zone });
},
ServerGeneral::TerrainBlockUpdates(blocks) => {
let blocks = blocks.decompress()?;
return Ok(TerrainUpdate::Block(blocks));
},
_ => {},
}
return Err(Error::Other("Invalid terrain message".into()));
}
fn handle_terrain_msg(&mut self, msg: TerrainUpdate) {
match msg {
TerrainUpdate::Chunk { key, chunk } => {
if let Some(chunk) = chunk {
self.state.insert_chunk(key, chunk);
}
self.pending_chunks.remove(&key);
},
ServerGeneral::LodZoneUpdate { key, zone } => {
TerrainUpdate::LodZone { key, zone } => {
self.lod_zones.insert(key, zone);
self.lod_last_requested = None;
},
ServerGeneral::TerrainBlockUpdates(blocks) => {
if let Some(mut blocks) = blocks.decompress() {
blocks.drain().for_each(|(pos, block)| {
TerrainUpdate::Block(blocks) => {
blocks.into_iter().for_each(|(pos, block)| {
self.state.set_block(pos, block);
});
}
},
_ => unreachable!("Not a terrain message"),
}
Ok(())
}
fn handle_server_character_screen_msg(
@ -2375,15 +2417,28 @@ impl Client {
}
while let Some(msg) = self.terrain_stream.try_recv_raw()? {
cnt += 1;
let terrain_tx = self.terrain_tx.clone();
self
.state
.slow_job_pool()
.spawn("TERRAIN_DESERIALIZING", move || {
let handle_msg = || {
let msg = msg.decompress()?;
let msg = bincode::deserialize(&msg)?;
Self::handle_server_terrain_msg(msg)
};
terrain_tx.send(handle_msg());
});
}
while let Ok(msg) = self.terrain_rx.try_recv() {
let msg = msg?;
#[cfg(feature = "tracy")]
{
if let ServerGeneral::TerrainChunkUpdate { chunk, .. } = &msg {
if let TerrainUpdate::Chunk { chunk, .. } = &msg {
terrain_cnt += chunk.as_ref().map(|x| x.approx_len()).unwrap_or(0);
}
}
self.handle_server_terrain_msg(msg)?;
self.handle_terrain_msg(msg);
}
if cnt_start == cnt {

144
common/net/src/msg/compression.rs
View File

@ -4,19 +4,55 @@ use common::{
volumes::vol_grid_2d::VolGrid2d,
};
use hashbrown::HashMap;
use image::{ImageBuffer, ImageDecoder, ImageEncoder, Pixel};
use image::{ImageBuffer, ImageDecoder, ImageEncoder, ImageError, Pixel};
use num_traits::cast::FromPrimitive;
use serde::{Deserialize, Serialize};
use std::{
borrow::Cow,
fmt::Debug,
io::{Read, Write},
io::{Error as IoError, Read, Write},
marker::PhantomData,
};
use serde_with::{serde_as, Bytes};
use tracing::warn;
use vek::*;
/// Errors that can occur during decoding.
#[derive(Debug)]
pub enum DecodeError {
Deserialize(bincode::Error),
Image(ImageError),
Io(IoError)
}
impl ToString for DecodeError {
fn to_string(&self) -> String {
match self {
Self::Deserialize(error) => error.to_string(),
Self::Image(error) => error.to_string(),
Self::Io(error) => error.to_string(),
}
}
}
impl From<ImageError> for DecodeError {
fn from(error: ImageError) -> Self {
Self::Image(error)
}
}
impl From<IoError> for DecodeError {
fn from(error: IoError) -> Self {
Self::Io(error)
}
}
impl From<bincode::Error> for DecodeError {
fn from(error: bincode::Error) -> Self {
Self::Deserialize(error)
}
}
/// Wrapper for compressed, serialized data (for stuff that doesn't use the
/// default lz4 compression)
#[serde_as]
@ -59,15 +95,14 @@ impl<'a, T: Serialize> CompressedData<'a, T> {
}
impl<T: for<'a> Deserialize<'a>> CompressedData<'_, T> {
pub fn decompress(&self) -> Option<T> {
pub fn decompress(&self) -> Result<T, DecodeError> {
if self.compressed {
let mut uncompressed = Vec::with_capacity(self.data.len());
flate2::read::DeflateDecoder::new(&*self.data)
.read_to_end(&mut uncompressed)
.ok()?;
bincode::deserialize(&*uncompressed).ok()
.read_to_end(&mut uncompressed)?;
Ok(bincode::deserialize(&*uncompressed)?)
} else {
bincode::deserialize(&*self.data).ok()
Ok(bincode::deserialize(&*self.data)?)
}
}
}
@ -129,6 +164,8 @@ impl PackingFormula for GridLtrPacking {
pub trait VoxelImageEncoding {
type Workspace;
type Output;
type EncodeError;
fn create(width: u32, height: u32) -> Self::Workspace;
fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>);
fn put_sprite(
@ -140,25 +177,31 @@ pub trait VoxelImageEncoding {
sprite: SpriteKind,
ori: Option<u8>,
);
fn finish(ws: &Self::Workspace) -> Option<Self::Output>;
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError>;
}
pub trait VoxelImageDecoding: VoxelImageEncoding {
fn start(ws: &Self::Output) -> Option<Self::Workspace>;
fn start(ws: &Self::Output) -> Result<Self::Workspace, DecodeError>;
fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block;
}
pub fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u8>>(
decoder: I,
) -> Option<ImageBuffer<P, Vec<u8>>> {
) -> Result<ImageBuffer<P, Vec<u8>>, ImageError> {
let (w, h) = decoder.dimensions();
let mut buf = vec![0; decoder.total_bytes() as usize];
decoder.read_image(&mut buf).ok()?;
ImageBuffer::from_raw(w, h, buf)
decoder.read_image(&mut buf)?;
Ok(ImageBuffer::from_raw(w, h, buf).ok_or_else(|| {
IoError::new(
std::io::ErrorKind::WriteZero,
"Could not write full image to buffer; maybe out of memory?"
)
})?)
}
impl<'a, VIE: VoxelImageEncoding> VoxelImageEncoding for &'a VIE {
type Output = VIE::Output;
type EncodeError = VIE::EncodeError;
type Workspace = VIE::Workspace;
fn create(width: u32, height: u32) -> Self::Workspace { VIE::create(width, height) }
@ -179,11 +222,11 @@ impl<'a, VIE: VoxelImageEncoding> VoxelImageEncoding for &'a VIE {
(*self).put_sprite(ws, x, y, kind, sprite, ori)
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> { VIE::finish(ws) }
fn finish(ws: &Self::Workspace) -> Result<Self::Output, VIE::EncodeError> { VIE::finish(ws) }
}
impl<'a, VIE: VoxelImageDecoding> VoxelImageDecoding for &'a VIE {
fn start(ws: &Self::Output) -> Option<Self::Workspace> { VIE::start(ws) }
fn start(ws: &Self::Output) -> Result<Self::Workspace, DecodeError> { VIE::start(ws) }
fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block {
VIE::get_block(ws, x, y, is_border)
@ -195,6 +238,7 @@ pub struct QuadPngEncoding<'a, const RESOLUTION_DIVIDER: u32>(pub PhantomData<&'
impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
type Output = CompressedData<'a, (Vec<u8>, [usize; 3])>;
type EncodeError = ImageError;
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
@ -232,20 +276,19 @@ impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
indices[i] = buf.len();
Some(())
Ok(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
@ -262,11 +305,10 @@ impl<'a, const N: u32> VoxelImageEncoding for QuadPngEncoding<'a, N> {
ws.3.width(),
ws.3.height(),
image::ColorType::Rgb8,
)
.ok()?;
)?;
}
Some(CompressedData::compress(&(buf, indices), 4))
Ok(CompressedData::compress(&(buf, indices), 4))
}
}
@ -323,7 +365,7 @@ const fn gen_lanczos_lookup<const N: u32, const R: u32>(
}
impl<const N: u32> VoxelImageDecoding for QuadPngEncoding<'_, N> {
fn start(data: &Self::Output) -> Option<Self::Workspace> {
fn start(data: &Self::Output) -> Result<Self::Workspace, DecodeError> {
use image::codecs::png::PngDecoder;
let (quad, indices) = data.decompress()?;
let ranges: [_; 4] = [
@ -332,11 +374,11 @@ impl<const N: u32> VoxelImageDecoding for QuadPngEncoding<'_, N> {
indices[1]..indices[2],
indices[2]..quad.len(),
];
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
let d = image_from_bytes(PngDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
Some((a, b, c, d))
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
let d = image_from_bytes(PngDecoder::new(&quad[ranges[3].clone()])?)?;
Ok((a, b, c, d))
}
fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block {
@ -469,6 +511,7 @@ pub struct TriPngEncoding<'a, const AVERAGE_PALETTE: bool>(pub PhantomData<&'a (
impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a, AVERAGE_PALETTE> {
type Output = CompressedData<'a, (Vec<u8>, Vec<Rgb<u8>>, [usize; 3])>;
type EncodeError = ImageError;
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
@ -508,20 +551,19 @@ impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a,
ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
indices[i] = buf.len();
Some(())
Ok(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
@ -550,12 +592,12 @@ impl<'a, const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<'a,
Vec::new()
};
Some(CompressedData::compress(&(buf, palette, indices), 4))
Ok(CompressedData::compress(&(buf, palette, indices), 4))
}
}
impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVERAGE_PALETTE> {
fn start(data: &Self::Output) -> Option<Self::Workspace> {
fn start(data: &Self::Output) -> Result<Self::Workspace, DecodeError> {
use image::codecs::png::PngDecoder;
let (quad, palette, indices) = data.decompress()?;
let ranges: [_; 3] = [
@ -563,9 +605,9 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVER
indices[0]..indices[1],
indices[1]..indices[2],
];
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
let mut d: HashMap<_, HashMap<_, _>> = HashMap::new();
if AVERAGE_PALETTE {
for i in 0..=255 {
@ -578,7 +620,7 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<'_, AVER
}
}
Some((a, b, c, d))
Ok((a, b, c, d))
}
fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
@ -681,7 +723,7 @@ pub fn image_terrain_chonk<S: RectVolSize, Storage: core::ops::DerefMut<Target=V
vie: &VIE,
packing: P,
chonk: &Chonk<Block, Storage, S, M>,
) -> Option<VIE::Output> {
) -> Result<VIE::Output, VIE::EncodeError> {
image_terrain(
vie,
packing,
@ -701,7 +743,7 @@ pub fn image_terrain_volgrid<
vie: &VIE,
packing: P,
volgrid: &VolGrid2d<Chonk<Block, Storage, S, M>>,
) -> Option<VIE::Output> {
) -> Result<VIE::Output, VIE::EncodeError> {
let mut lo = Vec3::broadcast(i32::MAX);
let mut hi = Vec3::broadcast(i32::MIN);
for (pos, chonk) in volgrid.iter() {
@ -727,7 +769,7 @@ pub fn image_terrain<
vol: &V,
lo: Vec3<u32>,
hi: Vec3<u32>,
) -> Option<VIE::Output> {
) -> Result<VIE::Output, VIE::EncodeError> {
let dims = Vec3::new(
hi.x.wrapping_sub(lo.x),
hi.y.wrapping_sub(lo.y),
@ -782,7 +824,7 @@ pub fn write_image_terrain<
data: &VIE::Output,
lo: Vec3<u32>,
hi: Vec3<u32>,
) -> Option<()> {
) -> Result<(), DecodeError> {
let ws = VIE::start(data)?;
let dims = Vec3::new(
hi.x.wrapping_sub(lo.x),
@ -805,7 +847,7 @@ pub fn write_image_terrain<
}
}
}
Some(())
Ok(())
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -824,16 +866,14 @@ pub struct WireChonk<VIE: VoxelImageEncoding, P: PackingFormula, M: Clone, S: Re
impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone, S: RectVolSize>
WireChonk<VIE, P, M, S>
{
pub fn from_chonk<Storage: core::ops::DerefMut<Target=Vec<Block>>>(vie: VIE, packing: P, chonk: &Chonk<Block, Storage, S, M>) -> Option<Self> {
pub fn from_chonk<Storage: core::ops::DerefMut<Target=Vec<Block>>>(vie: VIE, packing: P, chonk: &Chonk<Block, Storage, S, M>) -> Result<Self, VIE::EncodeError> {
let data = image_terrain_chonk(&vie, packing, chonk)?;
Some(Self {
Ok(Self {
zmin: chonk.get_min_z(),
zmax: chonk.get_max_z(),
data,
below: *chonk
.get(Vec3::new(0, 0, chonk.get_min_z().saturating_sub(1)))
.ok()?,
above: *chonk.get(Vec3::new(0, 0, chonk.get_max_z() + 1)).ok()?,
below: *chonk.below(),
above: *chonk.above(),
meta: chonk.meta().clone(),
vie,
packing,
@ -841,7 +881,7 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
})
}
pub fn to_chonk<Storage: Clone + core::ops::DerefMut<Target=Vec<Block>> + From<Vec<Block>>>(&self) -> Option<Chonk<Block, Storage, S, M>> {
pub fn to_chonk<Storage: Clone + core::ops::DerefMut<Target=Vec<Block>> + From<Vec<Block>>>(&self) -> Result<Chonk<Block, Storage, S, M>, DecodeError> {
let mut chonk = Chonk::new(self.zmin, self.below, self.above, self.meta.clone());
write_image_terrain(
&self.vie,
@ -851,6 +891,6 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
Vec3::new(0, 0, self.zmin as u32),
Vec3::new(S::RECT_SIZE.x, S::RECT_SIZE.y, self.zmax as u32),
)?;
Some(chonk)
Ok(chonk)
}
}

2
common/net/src/msg/mod.rs
View File

@ -8,7 +8,7 @@ pub mod world_msg;
pub use self::{
client::{ClientGeneral, ClientMsg, ClientRegister, ClientType},
compression::{
CompressedData, GridLtrPacking, PackingFormula, QuadPngEncoding, TriPngEncoding,
CompressedData, DecodeError, GridLtrPacking, PackingFormula, QuadPngEncoding, TriPngEncoding,
VoxelImageEncoding, WidePacking, WireChonk,
},
ecs_packet::EcsCompPacket,

22
common/net/src/msg/server.rs
View File

@ -1,5 +1,5 @@
use super::{
world_msg::EconomyInfo, ClientType, CompressedData, EcsCompPacket, PingMsg, QuadPngEncoding,
world_msg::EconomyInfo, ClientType, CompressedData, DecodeError, EcsCompPacket, PingMsg, QuadPngEncoding,
TriPngEncoding, WidePacking, WireChonk,
};
use crate::sync;
@ -104,24 +104,26 @@ impl SerializedTerrainChunk<'_> {
}
pub fn quadpng(chunk: &TerrainChunk) -> Self {
if let Some(wc) = WireChonk::from_chonk(QuadPngEncoding(PhantomData), WidePacking(), chunk) {
Self::QuadPng(wc)
} else {
warn!("Image encoding failure occurred, falling back to deflate");
match WireChonk::from_chonk(QuadPngEncoding(PhantomData), WidePacking(), chunk) {
Ok(wc) => Self::QuadPng(wc),
Err(err) => {
warn!("Image encoding failure occurred, falling back to deflate: {}", err);
Self::deflate(chunk)
}
}
}
pub fn tripng(chunk: &TerrainChunk) -> Self {
if let Some(wc) = WireChonk::from_chonk(TriPngEncoding(PhantomData), WidePacking(), chunk) {
Self::TriPng(wc)
} else {
warn!("Image encoding failure occurred, falling back to deflate");
match WireChonk::from_chonk(TriPngEncoding(PhantomData), WidePacking(), chunk) {
Ok(wc) => Self::TriPng(wc),
Err(err) => {
warn!("Image encoding failure occurred, falling back to deflate: {}", err);
Self::deflate(chunk)
},
}
}
pub fn to_chunk(&self) -> Option<TerrainChunk> {
pub fn to_chunk(&self) -> Result<TerrainChunk, DecodeError> {
match self {
Self::DeflatedChonk(chonk) => chonk.decompress(),
Self::QuadPng(wc) => wc.to_chonk(),

4
common/src/terrain/chonk.rs
View File

@ -84,6 +84,10 @@ impl<V, Storage: core::ops::DerefMut<Target=Vec<V>>, S: RectVolSize, M: Clone> C
}
}
pub fn below(&self) -> &V { &self.below }
pub fn above(&self) -> &V { &self.above }
pub fn meta(&self) -> &M { &self.meta }
#[inline]

1
voxygen/src/menu/main/mod.rs
View File

@ -432,6 +432,7 @@ fn get_client_msg_error(
localization.get("main.login.authentication_error"),
e
),
Error::CompressionError(e) => net_error(e.to_string(), mismatched_server_info),
Error::Kicked(e) => e,
Error::TooManyPlayers => localization.get("main.login.server_full").into(),
Error::AuthServerNotTrusted => {

81
world/examples/chunk_compression_benchmarks.rs
View File

@ -9,13 +9,13 @@ use common::{
},
};
use common_net::msg::compression::{
image_from_bytes, image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking,
image_from_bytes, image_terrain_chonk, image_terrain_volgrid, CompressedData, DecodeError, GridLtrPacking,
PackingFormula, QuadPngEncoding, TriPngEncoding, VoxelImageDecoding, VoxelImageEncoding,
WidePacking,
};
use core::marker::PhantomData;
use hashbrown::HashMap;
use image::{ImageBuffer, ImageEncoder};
use image::{ImageBuffer, ImageEncoder, ImageError};
use num_traits::cast::FromPrimitive;
use rayon::ThreadPoolBuilder;
use serde::{Deserialize, Serialize};
@ -166,6 +166,7 @@ pub struct PngEncoding;
impl VoxelImageEncoding for PngEncoding {
type Output = Vec<u8>;
type EncodeError = ImageError;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
@ -193,7 +194,7 @@ impl VoxelImageEncoding for PngEncoding {
);
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
use image::codecs::png::{CompressionType, FilterType};
let mut buf = Vec::new();
let png = image::codecs::png::PngEncoder::new_with_quality(
@ -206,9 +207,8 @@ impl VoxelImageEncoding for PngEncoding {
ws.width(),
ws.height(),
image::ColorType::Rgba8,
)
.ok()?;
Some(buf)
)?;
Ok(buf)
}
}
@ -217,6 +217,7 @@ pub struct JpegEncoding;
impl VoxelImageEncoding for JpegEncoding {
type Output = Vec<u8>;
type EncodeError = ImageError;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
@ -240,11 +241,11 @@ impl VoxelImageEncoding for JpegEncoding {
ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
jpeg.encode_image(ws).ok()?;
Some(buf)
jpeg.encode_image(ws)?;
Ok(buf)
}
}
@ -253,6 +254,7 @@ pub struct MixedEncoding;
impl VoxelImageEncoding for MixedEncoding {
type Output = (Vec<u8>, [usize; 3]);
type EncodeError = ImageError;
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
@ -291,33 +293,32 @@ impl VoxelImageEncoding for MixedEncoding {
ws.3.put_pixel(x, y, image::Rgb([0; 3]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
indices[i] = buf.len();
Some(())
Ok(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
f(&ws.2, 2)?;
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 10);
jpeg.encode_image(&ws.3).ok()?;
Some((buf, indices))
jpeg.encode_image(&ws.3)?;
Ok((buf, indices))
}
}
impl VoxelImageDecoding for MixedEncoding {
fn start((quad, indices): &Self::Output) -> Option<Self::Workspace> {
fn start((quad, indices): &Self::Output) -> Result<Self::Workspace, DecodeError> {
use image::codecs::{jpeg::JpegDecoder, png::PngDecoder};
let ranges: [_; 4] = [
0..indices[0],
@ -325,11 +326,11 @@ impl VoxelImageDecoding for MixedEncoding {
indices[1]..indices[2],
indices[2]..quad.len(),
];
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
Some((a, b, c, d))
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()])?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()])?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()])?)?;
let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()])?)?;
Ok((a, b, c, d))
}
fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
@ -366,6 +367,7 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
usize,
CompressedData<'static, HashMap<Vec2<u32>, (SpriteKind, u8)>>,
);
type EncodeError = ImageError;
type Workspace = (
image::ImageBuffer<image::Luma<u8>, Vec<u8>>,
image::ImageBuffer<image::Rgb<u8>, Vec<u8>>,
@ -399,7 +401,7 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
ws.2.insert(Vec2::new(x, y), (sprite, ori.unwrap_or(0)));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let png = image::codecs::png::PngEncoder::new_with_quality(
@ -412,12 +414,11 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
ws.0.width(),
ws.0.height(),
image::ColorType::L8,
)
.ok()?;
)?;
let index = buf.len();
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
jpeg.encode_image(&ws.1).ok()?;
Some((buf, index, CompressedData::compress(&ws.2, 4)))
jpeg.encode_image(&ws.1)?;
Ok((buf, index, CompressedData::compress(&ws.2, 4)))
}
}
@ -426,6 +427,7 @@ pub struct MixedEncodingDenseSprites;
impl VoxelImageEncoding for MixedEncodingDenseSprites {
type Output = (Vec<u8>, [usize; 3]);
type EncodeError = ImageError;
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
Vec<u8>,
@ -462,20 +464,19 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
ws.3.put_pixel(x, y, image::Rgb([0; 3]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Fast,
FilterType::Up,
);
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
indices[i] = buf.len();
Some(())
Ok(())
};
f(&ws.0, 0)?;
let mut g = |x: &[u8], i| {
@ -487,8 +488,8 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
g(&ws.2, 2);
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
jpeg.encode_image(&ws.3).ok()?;
Some((buf, indices))
jpeg.encode_image(&ws.3)?;
Ok((buf, indices))
}
}
@ -592,6 +593,7 @@ pub struct PaletteEncoding<'a, NN: NearestNeighbor, const N: u32>(&'a HashMap<Bl
impl<'a, NN: NearestNeighbor, const N: u32> VoxelImageEncoding for PaletteEncoding<'a, NN, N> {
type Output = CompressedData<'a, (Vec<u8>, [usize; 4])>;
type EncodeError = ImageError;
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
@ -628,27 +630,26 @@ impl<'a, NN: NearestNeighbor, const N: u32> VoxelImageEncoding for PaletteEncodi
ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
fn finish(ws: &Self::Workspace) -> Result<Self::Output, Self::EncodeError> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 4];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| -> Result<_, Self::EncodeError> {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
png.write_image(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)?;
indices[i] = buf.len();
Some(())
Ok(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
f(&ws.2, 2)?;
f(&ws.3, 3)?;
Some(CompressedData::compress(&(buf, indices), 1))
Ok(CompressedData::compress(&(buf, indices), 1))
}
}

2
world/src/civ/mod.rs
View File

@ -1347,7 +1347,7 @@ impl SiteKind {
},
SiteKind::Citadel => (-0.3..0.7).contains(&chunk.temp) && chunk.tree_density < 0.4,
SiteKind::GiantTree/* | SiteKind::Tree*/ => {
chunk.tree_density > 0.4 && (-0.3..0.4).contains(&chunk.temp)
chunk.tree_density > 0.4/* && (-0.3..0.4).contains(&chunk.temp) */
},
SiteKind::CliffTown => {
(-0.6..0.4).contains(&chunk.temp)