diff --git a/Cargo.lock b/Cargo.lock index cb6680eef9..9017f2f008 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5515,6 +5515,7 @@ dependencies = [ "bincode", "flate2", "hashbrown", + "image", "serde", "specs", "specs-idvs", diff --git a/common/net/Cargo.toml b/common/net/Cargo.toml index 0661f79697..3c0905f351 100644 --- a/common/net/Cargo.toml +++ b/common/net/Cargo.toml @@ -16,6 +16,7 @@ common = {package = "veloren-common", path = "../../common"} bincode = "1.3.3" flate2 = "1.0.20" +image = { version = "0.23.12", default-features = false, features = ["png", "jpeg"] } sum_type = "0.2.0" vek = { version = "=0.14.1", features = ["serde"] } tracing = { version = "0.1", default-features = false } diff --git a/common/net/src/msg/compression.rs b/common/net/src/msg/compression.rs new file mode 100644 index 0000000000..2196ee40c2 --- /dev/null +++ b/common/net/src/msg/compression.rs @@ -0,0 +1,407 @@ +use common::{ + terrain::{chonk::Chonk, Block, BlockKind, SpriteKind}, + vol::{BaseVol, IntoVolIterator, ReadVol, RectVolSize, SizedVol, WriteVol}, + volumes::vol_grid_2d::VolGrid2d, +}; +use hashbrown::HashMap; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::Debug, + io::{Read, Write}, + marker::PhantomData, +}; +use tracing::trace; +use vek::*; + +/// Wrapper for compressed, serialized data (for stuff that doesn't use the +/// default lz4 compression) +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CompressedData { + pub data: Vec, + compressed: bool, + _phantom: PhantomData, +} + +impl CompressedData { + pub fn compress(t: &T, level: u32) -> Self { + use flate2::{write::DeflateEncoder, Compression}; + let uncompressed = bincode::serialize(t) + .expect("bincode serialization can only fail if a byte limit is set"); + + if uncompressed.len() >= 32 { + const EXPECT_MSG: &str = + "compression only fails for fallible Read/Write impls (which Vec is not)"; + + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(level)); + encoder.write_all(&*uncompressed).expect(EXPECT_MSG); + let compressed = encoder.finish().expect(EXPECT_MSG); + trace!( + "compressed {}, uncompressed {}, ratio {}", + compressed.len(), + uncompressed.len(), + compressed.len() as f32 / uncompressed.len() as f32 + ); + CompressedData { + data: compressed, + compressed: true, + _phantom: PhantomData, + } + } else { + CompressedData { + data: uncompressed, + compressed: false, + _phantom: PhantomData, + } + } + } +} + +impl Deserialize<'a>> CompressedData { + pub fn decompress(&self) -> Option { + if self.compressed { + let mut uncompressed = Vec::new(); + flate2::read::DeflateDecoder::new(&*self.data) + .read_to_end(&mut uncompressed) + .ok()?; + bincode::deserialize(&*uncompressed).ok() + } else { + bincode::deserialize(&*self.data).ok() + } + } +} + +/// Formula for packing voxel data into a 2d array +pub trait PackingFormula { + fn dimensions(&self, dims: Vec3) -> (u32, u32); + fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32); +} + +/// A tall, thin image, with no wasted space, but which most image viewers don't +/// handle well. Z levels increase from top to bottom, xy-slices are stacked +/// vertically. +pub struct TallPacking { + /// Making the borders go back and forth based on z-parity preserves spatial + /// locality better, but is more confusing to look at + pub flip_y: bool, +} + +impl PackingFormula for TallPacking { + fn dimensions(&self, dims: Vec3) -> (u32, u32) { (dims.x, dims.y * dims.z) } + + fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32) { + let i = x; + let j0 = if self.flip_y { + if z % 2 == 0 { y } else { dims.y - y - 1 } + } else { + y + }; + let j = z * dims.y + j0; + (i, j) + } +} + +/// A grid of the z levels, left to right, top to bottom, like English prose. +/// Convenient for visualizing terrain, but wastes space if the number of z +/// levels isn't a perfect square. +pub struct GridLtrPacking; + +impl PackingFormula for GridLtrPacking { + fn dimensions(&self, dims: Vec3) -> (u32, u32) { + let rootz = (dims.z as f64).sqrt().ceil() as u32; + (dims.x * rootz, dims.y * rootz) + } + + fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32) { + let rootz = (dims.z as f64).sqrt().ceil() as u32; + let i = x + (z % rootz) * dims.x; + let j = y + (z / rootz) * dims.y; + (i, j) + } +} + +pub trait VoxelImageEncoding { + type Workspace; + type Output; + fn create(width: u32, height: u32) -> Self::Workspace; + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb); + fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind, ori: Option); + fn finish(ws: &Self::Workspace) -> Self::Output; +} + +pub struct PngEncoding; + +impl VoxelImageEncoding for PngEncoding { + type Output = Vec; + type Workspace = image::ImageBuffer, Vec>; + + fn create(width: u32, height: u32) -> Self::Workspace { + use image::{ImageBuffer, Rgba}; + ImageBuffer::, Vec>::new(width, height) + } + + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { + ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8])); + } + + fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind, ori: Option) { + ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, ori.unwrap_or(0), 255])); + } + + fn finish(ws: &Self::Workspace) -> Self::Output { + use image::codecs::png::{CompressionType, FilterType}; + let mut buf = Vec::new(); + let png = image::codecs::png::PngEncoder::new_with_quality( + &mut buf, + CompressionType::Fast, + FilterType::Up, + ); + png.encode( + &*ws.as_raw(), + ws.width(), + ws.height(), + image::ColorType::Rgba8, + ) + .unwrap(); + buf + } +} + +pub struct JpegEncoding; + +impl VoxelImageEncoding for JpegEncoding { + type Output = Vec; + type Workspace = image::ImageBuffer, Vec>; + + fn create(width: u32, height: u32) -> Self::Workspace { + use image::{ImageBuffer, Rgba}; + ImageBuffer::, Vec>::new(width, height) + } + + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { + ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8])); + } + + fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind, _: Option) { + ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255])); + } + + fn finish(ws: &Self::Workspace) -> Self::Output { + let mut buf = Vec::new(); + let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1); + jpeg.encode_image(ws).unwrap(); + buf + } +} + +pub struct MixedEncoding; + +impl VoxelImageEncoding for MixedEncoding { + type Output = (Vec, [usize; 3]); + type Workspace = ( + image::ImageBuffer, Vec>, + image::ImageBuffer, Vec>, + image::ImageBuffer, Vec>, + image::ImageBuffer, Vec>, + ); + + fn create(width: u32, height: u32) -> Self::Workspace { + use image::ImageBuffer; + ( + ImageBuffer::new(width, height), + ImageBuffer::new(width, height), + ImageBuffer::new(width, height), + ImageBuffer::new(width, height), + ) + } + + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.1.put_pixel(x, y, image::Luma([0])); + ws.2.put_pixel(x, y, image::Luma([0])); + ws.3.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b])); + } + + fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind, ori: Option) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.1.put_pixel(x, y, image::Luma([sprite as u8])); + ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)])); + ws.3.put_pixel(x, y, image::Rgb([0; 3])); + } + + fn finish(ws: &Self::Workspace) -> Self::Output { + let mut buf = Vec::new(); + use image::codecs::png::{CompressionType, FilterType}; + let mut indices = [0; 3]; + let mut f = |x: &image::ImageBuffer<_, Vec>, i| { + let png = image::codecs::png::PngEncoder::new_with_quality( + &mut buf, + CompressionType::Fast, + FilterType::Up, + ); + png.encode( + &*x.as_raw(), + x.width(), + x.height(), + image::ColorType::L8, + ) + .unwrap(); + indices[i] = buf.len(); + }; + f(&ws.0, 0); + f(&ws.1, 1); + f(&ws.2, 2); + + let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1); + jpeg.encode_image(&ws.3).unwrap(); + (buf, indices) + } +} + +pub fn image_terrain_chonk( + vie: VIE, + packing: P, + chonk: &Chonk, +) -> VIE::Output { + image_terrain( + vie, + packing, + chonk, + Vec3::new(0, 0, chonk.get_min_z() as u32), + Vec3::new(S::RECT_SIZE.x, S::RECT_SIZE.y, chonk.get_max_z() as u32), + ) +} + +pub fn image_terrain_volgrid< + S: RectVolSize + Debug, + M: Clone + Debug, + P: PackingFormula, + VIE: VoxelImageEncoding, +>( + vie: VIE, + packing: P, + volgrid: &VolGrid2d>, +) -> VIE::Output { + let mut lo = Vec3::broadcast(i32::MAX); + let mut hi = Vec3::broadcast(i32::MIN); + for (pos, chonk) in volgrid.iter() { + lo.x = lo.x.min(pos.x * S::RECT_SIZE.x as i32); + lo.y = lo.y.min(pos.y * S::RECT_SIZE.y as i32); + lo.z = lo.z.min(chonk.get_min_z()); + + hi.x = hi.x.max((pos.x + 1) * S::RECT_SIZE.x as i32); + hi.y = hi.y.max((pos.y + 1) * S::RECT_SIZE.y as i32); + hi.z = hi.z.max(chonk.get_max_z()); + } + + image_terrain(vie, packing, volgrid, lo.as_(), hi.as_()) +} + +pub fn image_terrain< + V: BaseVol + ReadVol, + P: PackingFormula, + VIE: VoxelImageEncoding, +>( + _: VIE, + packing: P, + vol: &V, + lo: Vec3, + hi: Vec3, +) -> VIE::Output { + let dims = hi - lo; + + let (width, height) = packing.dimensions(dims); + let mut image = VIE::create(width, height); + for z in 0..dims.z { + for y in 0..dims.y { + for x in 0..dims.x { + let (i, j) = packing.index(dims, x, y, z); + + let block = *vol + .get(Vec3::new(x + lo.x, y + lo.y, z + lo.z).as_()) + .unwrap_or(&Block::empty()); + match (block.get_color(), block.get_sprite()) { + (Some(rgb), None) => { + VIE::put_solid(&mut image, i, j, *block, rgb); + }, + (None, Some(sprite)) => { + VIE::put_sprite(&mut image, i, j, *block, sprite, block.get_ori()); + }, + _ => panic!( + "attr being used for color vs sprite is mutually exclusive (and that's \ + required for this translation to be lossless), but there's no way to \ + guarantee that at the type level with Block's public API" + ), + } + } + } + } + + VIE::finish(&image) +} + +pub struct MixedEncodingDenseSprites; + +impl VoxelImageEncoding for MixedEncodingDenseSprites { + type Output = (Vec, [usize; 3]); + type Workspace = ( + image::ImageBuffer, Vec>, + Vec, + Vec, + image::ImageBuffer, Vec>, + ); + + fn create(width: u32, height: u32) -> Self::Workspace { + use image::ImageBuffer; + ( + ImageBuffer::new(width, height), + Vec::new(), + Vec::new(), + ImageBuffer::new(width, height), + ) + } + + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.3.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b])); + } + + fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind, ori: Option) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.1.push(sprite as u8); + ws.2.push(ori.unwrap_or(0)); + ws.3.put_pixel(x, y, image::Rgb([0; 3])); + } + + fn finish(ws: &Self::Workspace) -> Self::Output { + let mut buf = Vec::new(); + use image::codecs::png::{CompressionType, FilterType}; + let mut indices = [0; 3]; + let mut f = |x: &image::ImageBuffer<_, Vec>, i| { + let png = image::codecs::png::PngEncoder::new_with_quality( + &mut buf, + CompressionType::Fast, + FilterType::Up, + ); + png.encode( + &*x.as_raw(), + x.width(), + x.height(), + image::ColorType::L8, + ) + .unwrap(); + indices[i] = buf.len(); + }; + f(&ws.0, 0); + let mut g = |x: &[u8], i| { + buf.extend_from_slice(&*CompressedData::compress(&x, 4).data); + indices[i] = buf.len(); + }; + + g(&ws.1, 1); + g(&ws.2, 2); + + let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1); + jpeg.encode_image(&ws.3).unwrap(); + (buf, indices) + } +} diff --git a/common/net/src/msg/mod.rs b/common/net/src/msg/mod.rs index 2d2421f9cc..25d315da2d 100644 --- a/common/net/src/msg/mod.rs +++ b/common/net/src/msg/mod.rs @@ -1,4 +1,5 @@ pub mod client; +pub mod compression; pub mod ecs_packet; pub mod server; pub mod world_msg; @@ -6,6 +7,10 @@ pub mod world_msg; // Reexports pub use self::{ client::{ClientGeneral, ClientMsg, ClientRegister, ClientType}, + compression::{ + CompressedData, GridLtrPacking, JpegEncoding, MixedEncoding, PackingFormula, PngEncoding, + TallPacking, VoxelImageEncoding, + }, ecs_packet::EcsCompPacket, server::{ CharacterInfo, DisconnectReason, InviteAnswer, Notification, PlayerInfo, PlayerListUpdate, @@ -15,8 +20,6 @@ pub use self::{ }; use common::character::CharacterId; use serde::{Deserialize, Serialize}; -use std::marker::PhantomData; -use tracing::trace; #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub enum PresenceKind { @@ -44,60 +47,3 @@ pub fn validate_chat_msg(msg: &str) -> Result<(), ChatMsgValidationError> { Err(ChatMsgValidationError::TooLong) } } - -/// Wrapper for compressed, serialized data (for stuff that doesn't use the -/// default lz4 compression) -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompressedData { - pub data: Vec, - compressed: bool, - _phantom: PhantomData, -} - -impl Deserialize<'a>> CompressedData { - pub fn compress(t: &T, level: u32) -> Self { - use flate2::{write::DeflateEncoder, Compression}; - use std::io::Write; - let uncompressed = bincode::serialize(t) - .expect("bincode serialization can only fail if a byte limit is set"); - - if uncompressed.len() >= 32 { - const EXPECT_MSG: &str = - "compression only fails for fallible Read/Write impls (which Vec is not)"; - - let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(level)); - encoder.write_all(&*uncompressed).expect(EXPECT_MSG); - let compressed = encoder.finish().expect(EXPECT_MSG); - trace!( - "compressed {}, uncompressed {}, ratio {}", - compressed.len(), - uncompressed.len(), - compressed.len() as f32 / uncompressed.len() as f32 - ); - CompressedData { - data: compressed, - compressed: true, - _phantom: PhantomData, - } - } else { - CompressedData { - data: uncompressed, - compressed: false, - _phantom: PhantomData, - } - } - } - - pub fn decompress(&self) -> Option { - use std::io::Read; - if self.compressed { - let mut uncompressed = Vec::new(); - flate2::read::DeflateDecoder::new(&*self.data) - .read_to_end(&mut uncompressed) - .ok()?; - bincode::deserialize(&*uncompressed).ok() - } else { - bincode::deserialize(&*self.data).ok() - } - } -} diff --git a/world/src/bin/chunk_compression_benchmarks.rs b/world/src/bin/chunk_compression_benchmarks.rs index e1d0c9bb23..03541fb8d7 100644 --- a/world/src/bin/chunk_compression_benchmarks.rs +++ b/world/src/bin/chunk_compression_benchmarks.rs @@ -7,9 +7,13 @@ use common::{ vol_grid_2d::VolGrid2d, }, }; +use common_net::msg::compression::{ + image_terrain, image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking, + JpegEncoding, MixedEncoding, MixedEncodingDenseSprites, PackingFormula, PngEncoding, + TallPacking, VoxelImageEncoding, +}; use hashbrown::HashMap; use std::{ - fmt::Debug, io::{Read, Write}, sync::Arc, time::Instant, @@ -17,6 +21,7 @@ use std::{ use tracing::{debug, trace}; use vek::*; use veloren_world::{ + civ::SiteKind, sim::{FileOpts, WorldOpts, DEFAULT_WORLD_MAP}, World, }; @@ -115,136 +120,18 @@ fn channelize_dyna( (blocks, r, g, b, sprites) } -/// Formula for packing voxel data into a 2d array -pub trait PackingFormula { - fn dimensions(&self, dims: Vec3) -> (u32, u32); - fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32); -} +pub struct MixedEncodingSparseSprites; -/// A tall, thin image, with no wasted space, but which most image viewers don't -/// handle well. Z levels increase from top to bottom, xy-slices are stacked -/// vertically. -pub struct TallPacking { - /// Making the borders go back and forth based on z-parity preserves spatial - /// locality better, but is more confusing to look at - pub flip_y: bool, -} - -impl PackingFormula for TallPacking { - fn dimensions(&self, dims: Vec3) -> (u32, u32) { (dims.x, dims.y * dims.z) } - - fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32) { - let i = x; - let j0 = if self.flip_y { - if z % 2 == 0 { y } else { dims.y - y - 1 } - } else { - y - }; - let j = z * dims.y + j0; - (i, j) - } -} - -/// A grid of the z levels, left to right, top to bottom, like English prose. -/// Convenient for visualizing terrain, but wastes space if the number of z -/// levels isn't a perfect square. -pub struct GridLtrPacking; - -impl PackingFormula for GridLtrPacking { - fn dimensions(&self, dims: Vec3) -> (u32, u32) { - let rootz = (dims.z as f64).sqrt().ceil() as u32; - (dims.x * rootz, dims.y * rootz) - } - - fn index(&self, dims: Vec3, x: u32, y: u32, z: u32) -> (u32, u32) { - let rootz = (dims.z as f64).sqrt().ceil() as u32; - let i = x + (z % rootz) * dims.x; - let j = y + (z / rootz) * dims.y; - (i, j) - } -} - -pub trait VoxelImageEncoding { - type Workspace; - type Output; - fn create(width: u32, height: u32) -> Self::Workspace; - fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb); - fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind); - fn finish(ws: &Self::Workspace) -> Self::Output; -} - -pub struct PngEncoding; - -impl VoxelImageEncoding for PngEncoding { - type Output = Vec; - type Workspace = image::ImageBuffer, Vec>; - - fn create(width: u32, height: u32) -> Self::Workspace { - use image::{ImageBuffer, Rgba}; - ImageBuffer::, Vec>::new(width, height) - } - - fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { - ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8])); - } - - fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind) { - ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255])); - } - - fn finish(ws: &Self::Workspace) -> Self::Output { - use image::codecs::png::{CompressionType, FilterType}; - let mut buf = Vec::new(); - let png = image::codecs::png::PngEncoder::new_with_quality( - &mut buf, - CompressionType::Fast, - FilterType::Up, - ); - png.encode( - &*ws.as_raw(), - ws.width(), - ws.height(), - image::ColorType::Rgba8, - ) - .unwrap(); - buf - } -} - -pub struct JpegEncoding; - -impl VoxelImageEncoding for JpegEncoding { - type Output = Vec; - type Workspace = image::ImageBuffer, Vec>; - - fn create(width: u32, height: u32) -> Self::Workspace { - use image::{ImageBuffer, Rgba}; - ImageBuffer::, Vec>::new(width, height) - } - - fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { - ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8])); - } - - fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind) { - ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255])); - } - - fn finish(ws: &Self::Workspace) -> Self::Output { - let mut buf = Vec::new(); - let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1); - jpeg.encode_image(ws).unwrap(); - buf - } -} - -pub struct MixedEncoding; - -impl VoxelImageEncoding for MixedEncoding { - type Output = (Vec, usize); +impl VoxelImageEncoding for MixedEncodingSparseSprites { + type Output = ( + Vec, + usize, + CompressedData, (SpriteKind, u8)>>, + ); type Workspace = ( - image::ImageBuffer, Vec>, + image::ImageBuffer, Vec>, image::ImageBuffer, Vec>, + HashMap, (SpriteKind, u8)>, ); fn create(width: u32, height: u32) -> Self::Workspace { @@ -252,17 +139,26 @@ impl VoxelImageEncoding for MixedEncoding { ( ImageBuffer::new(width, height), ImageBuffer::new(width, height), + HashMap::new(), ) } fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { - ws.0.put_pixel(x, y, image::LumaA([kind as u8, 0])); + ws.0.put_pixel(x, y, image::Luma([kind as u8])); ws.1.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b])); } - fn put_sprite(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, sprite: SpriteKind) { - ws.0.put_pixel(x, y, image::LumaA([kind as u8, sprite as u8])); + fn put_sprite( + ws: &mut Self::Workspace, + x: u32, + y: u32, + kind: BlockKind, + sprite: SpriteKind, + ori: Option, + ) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); ws.1.put_pixel(x, y, image::Rgb([0; 3])); + ws.2.insert(Vec2::new(x, y), (sprite, ori.unwrap_or(0))); } fn finish(ws: &Self::Workspace) -> Self::Output { @@ -277,92 +173,16 @@ impl VoxelImageEncoding for MixedEncoding { &*ws.0.as_raw(), ws.0.width(), ws.0.height(), - image::ColorType::La8, + image::ColorType::L8, ) .unwrap(); let index = buf.len(); let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1); jpeg.encode_image(&ws.1).unwrap(); - //println!("Mixed {} {}", index, buf.len()); - (buf, index) + (buf, index, CompressedData::compress(&ws.2, 4)) } } -fn image_terrain_chonk( - vie: VIE, - packing: P, - chonk: &Chonk, -) -> VIE::Output { - image_terrain( - vie, - packing, - chonk, - Vec3::new(0, 0, chonk.get_min_z() as u32), - Vec3::new(S::RECT_SIZE.x, S::RECT_SIZE.y, chonk.get_max_z() as u32), - ) -} - -fn image_terrain_volgrid< - S: RectVolSize + Debug, - M: Clone + Debug, - P: PackingFormula, - VIE: VoxelImageEncoding, ->( - vie: VIE, - packing: P, - volgrid: &VolGrid2d>, -) -> VIE::Output { - let mut lo = Vec3::broadcast(i32::MAX); - let mut hi = Vec3::broadcast(i32::MIN); - for (pos, chonk) in volgrid.iter() { - lo.x = lo.x.min(pos.x * S::RECT_SIZE.x as i32); - lo.y = lo.y.min(pos.y * S::RECT_SIZE.y as i32); - lo.z = lo.z.min(chonk.get_min_z()); - - hi.x = hi.x.max((pos.x + 1) * S::RECT_SIZE.x as i32); - hi.y = hi.y.max((pos.y + 1) * S::RECT_SIZE.y as i32); - hi.z = hi.z.max(chonk.get_max_z()); - } - println!("{:?} {:?}", lo, hi); - - image_terrain(vie, packing, volgrid, lo.as_(), hi.as_()) -} - -fn image_terrain + ReadVol, P: PackingFormula, VIE: VoxelImageEncoding>( - _: VIE, - packing: P, - vol: &V, - lo: Vec3, - hi: Vec3, -) -> VIE::Output { - let dims = hi - lo; - - let (width, height) = packing.dimensions(dims); - let mut image = VIE::create(width, height); - //println!("jpeg dims: {:?}", dims); - for z in 0..dims.z { - for y in 0..dims.y { - for x in 0..dims.x { - let (i, j) = packing.index(dims, x, y, z); - //println!("{:?} {:?}", (x, y, z), (i, j)); - - let block = *vol - .get(Vec3::new(x + lo.x, y + lo.y, z + lo.z).as_()) - .unwrap_or(&Block::empty()); - //println!("{} {} {} {:?}", x, y, z, block); - if let Some(rgb) = block.get_color() { - VIE::put_solid(&mut image, i, j, *block, rgb); - } else { - let sprite = block.get_sprite().unwrap(); - VIE::put_sprite(&mut image, i, j, *block, sprite); - } - } - } - } - - VIE::finish(&image) -} - fn histogram_to_dictionary(histogram: &HashMap, usize>, dictionary: &mut Vec) { let mut tmp: Vec<(Vec, usize)> = histogram.iter().map(|(k, v)| (k.clone(), *v)).collect(); tmp.sort_by_key(|(_, count)| *count); @@ -389,192 +209,295 @@ fn main() { ..WorldOpts::default() }); println!("Loaded world"); + const HISTOGRAMS: bool = false; let mut histogram: HashMap, usize> = HashMap::new(); let mut histogram2: HashMap, usize> = HashMap::new(); let mut dictionary = vec![0xffu8; 1 << 16]; let mut dictionary2 = vec![0xffu8; 1 << 16]; let k = 32; let sz = world.sim().get_size(); - let mut totals = [0.0; 10]; - let mut total_timings = [0.0; 7]; - let mut count = 0; - let mut volgrid = VolGrid2d::new().unwrap(); - for (i, (x, y)) in Spiral2d::new() - .radius(20) - .map(|v| (v.x + sz.x as i32 / 2, v.y + sz.y as i32 / 2)) - .enumerate() - { - let chunk = world.generate_chunk(index.as_index_ref(), Vec2::new(x as _, y as _), || false); - if let Ok((chunk, _)) = chunk { - let uncompressed = bincode::serialize(&chunk).unwrap(); - for w in uncompressed.windows(k) { - *histogram.entry(w.to_vec()).or_default() += 1; - } - if i % 128 == 0 { - histogram_to_dictionary(&histogram, &mut dictionary); - } - let lz4chonk_pre = Instant::now(); - let lz4_chonk = lz4_with_dictionary(&bincode::serialize(&chunk).unwrap(), &[]); - let lz4chonk_post = Instant::now(); - //let lz4_dict_chonk = SerializedTerrainChunk::from_chunk(&chunk, - // &*dictionary); - let deflatechonk_pre = Instant::now(); - let deflate_chonk = do_deflate_flate2(&bincode::serialize(&chunk).unwrap()); - let deflatechonk_post = Instant::now(); + let mut sites = Vec::new(); - let dyna: Dyna<_, _, ColumnAccess> = chonk_to_dyna(&chunk, Block::empty()); - let ser_dyna = bincode::serialize(&dyna).unwrap(); - for w in ser_dyna.windows(k) { - *histogram2.entry(w.to_vec()).or_default() += 1; - } - if i % 128 == 0 { - histogram_to_dictionary(&histogram2, &mut dictionary2); - } - let lz4_dyna = lz4_with_dictionary(&*ser_dyna, &[]); - //let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2); - let deflate_dyna = do_deflate(&*ser_dyna); - let deflate_channeled_dyna = - do_deflate_flate2(&bincode::serialize(&channelize_dyna(&dyna)).unwrap()); + sites.push(("center", sz / 2)); + sites.push(( + "dungeon", + world + .civs() + .sites() + .find(|s| s.is_dungeon()) + .map(|s| s.center.as_()) + .unwrap(), + )); + sites.push(( + "town", + world + .civs() + .sites() + .find(|s| s.is_settlement()) + .map(|s| s.center.as_()) + .unwrap(), + )); + sites.push(( + "castle", + world + .civs() + .sites() + .find(|s| s.is_castle()) + .map(|s| s.center.as_()) + .unwrap(), + )); + sites.push(( + "tree", + world + .civs() + .sites() + .find(|s| matches!(s.kind, SiteKind::Tree)) + .map(|s| s.center.as_()) + .unwrap(), + )); - let jpegchonkgrid_pre = Instant::now(); - let jpegchonkgrid = image_terrain_chonk(JpegEncoding, GridLtrPacking, &chunk); - let jpegchonkgrid_post = Instant::now(); - - if false { - use std::fs::File; - let mut f = File::create(&format!("chonkjpegs/tmp_{}_{}.jpg", x, y)).unwrap(); - f.write_all(&*jpegchonkgrid).unwrap(); - } - - let jpegchonktall_pre = Instant::now(); - let jpegchonktall = - image_terrain_chonk(JpegEncoding, TallPacking { flip_y: false }, &chunk); - let jpegchonktall_post = Instant::now(); - - let jpegchonkflip_pre = Instant::now(); - let jpegchonkflip = - image_terrain_chonk(JpegEncoding, TallPacking { flip_y: true }, &chunk); - let jpegchonkflip_post = Instant::now(); - - let mixedchonk_pre = Instant::now(); - let mixedchonk = - image_terrain_chonk(MixedEncoding, TallPacking { flip_y: true }, &chunk); - let mixedchonk_post = Instant::now(); - - let pngchonk_pre = Instant::now(); - let pngchonk = image_terrain_chonk(PngEncoding, GridLtrPacking, &chunk); - let pngchonk_post = Instant::now(); - - let n = uncompressed.len(); - let sizes = [ - lz4_chonk.len() as f32 / n as f32, - deflate_chonk.len() as f32 / n as f32, - lz4_dyna.len() as f32 / n as f32, - deflate_dyna.len() as f32 / n as f32, - deflate_channeled_dyna.len() as f32 / n as f32, - jpegchonkgrid.len() as f32 / n as f32, - jpegchonktall.len() as f32 / n as f32, - jpegchonkflip.len() as f32 / n as f32, - mixedchonk.0.len() as f32 / n as f32, - pngchonk.len() as f32 / n as f32, - ]; - let best_idx = sizes - .iter() - .enumerate() - .fold((1.0, 0), |(best, i), (j, ratio)| { - if ratio < &best { - (*ratio, j) - } else { - (best, i) + for (sitename, sitepos) in sites.iter() { + let mut totals = [0.0; 12]; + let mut total_timings = [0.0; 9]; + let mut count = 0; + let mut volgrid = VolGrid2d::new().unwrap(); + for (i, spiralpos) in Spiral2d::new() + .radius(7) + .map(|v| v + sitepos.as_()) + .enumerate() + { + let chunk = world.generate_chunk(index.as_index_ref(), spiralpos, || false); + if let Ok((chunk, _)) = chunk { + let uncompressed = bincode::serialize(&chunk).unwrap(); + if HISTOGRAMS { + for w in uncompressed.windows(k) { + *histogram.entry(w.to_vec()).or_default() += 1; } - }) - .1; - let timings = [ - (lz4chonk_post - lz4chonk_pre).subsec_nanos(), - (deflatechonk_post - deflatechonk_pre).subsec_nanos(), - (jpegchonkgrid_post - jpegchonkgrid_pre).subsec_nanos(), - (jpegchonktall_post - jpegchonktall_pre).subsec_nanos(), - (jpegchonkflip_post - jpegchonkflip_pre).subsec_nanos(), - (mixedchonk_post - mixedchonk_pre).subsec_nanos(), - (pngchonk_post - pngchonk_pre).subsec_nanos(), - ]; - trace!( - "{} {}: uncompressed: {}, {:?} {} {:?}", - x, - y, - n, - sizes, - best_idx, - timings - ); - for j in 0..totals.len() { - totals[j] += sizes[j]; - } - for j in 0..total_timings.len() { - total_timings[j] += timings[j] as f32; - } - count += 1; - let _ = volgrid.insert(Vec2::new(x, y), Arc::new(chunk)); + if i % 128 == 0 { + histogram_to_dictionary(&histogram, &mut dictionary); + } + } + let lz4chonk_pre = Instant::now(); + let lz4_chonk = lz4_with_dictionary(&bincode::serialize(&chunk).unwrap(), &[]); + let lz4chonk_post = Instant::now(); + //let lz4_dict_chonk = SerializedTerrainChunk::from_chunk(&chunk, + // &*dictionary); - if (1usize..10) - .into_iter() - .any(|i| (2 * i + 1) * (2 * i + 1) == count) - { - use std::fs::File; - let mut f = File::create(&format!("chonkjpegs/volgrid_{}.jpg", count)).unwrap(); - let jpeg_volgrid = image_terrain_volgrid(JpegEncoding, GridLtrPacking, &volgrid); - f.write_all(&*jpeg_volgrid).unwrap(); + let deflatechonk_pre = Instant::now(); + let deflate_chonk = do_deflate_flate2(&bincode::serialize(&chunk).unwrap()); + let deflatechonk_post = Instant::now(); + + let dyna: Dyna<_, _, ColumnAccess> = chonk_to_dyna(&chunk, Block::empty()); + let ser_dyna = bincode::serialize(&dyna).unwrap(); + if HISTOGRAMS { + for w in ser_dyna.windows(k) { + *histogram2.entry(w.to_vec()).or_default() += 1; + } + if i % 128 == 0 { + histogram_to_dictionary(&histogram2, &mut dictionary2); + } + } + let lz4_dyna = lz4_with_dictionary(&*ser_dyna, &[]); + //let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2); + let deflate_dyna = do_deflate(&*ser_dyna); + let deflate_channeled_dyna = + do_deflate_flate2(&bincode::serialize(&channelize_dyna(&dyna)).unwrap()); + + let jpegchonkgrid_pre = Instant::now(); + let jpegchonkgrid = image_terrain_chonk(JpegEncoding, GridLtrPacking, &chunk); + let jpegchonkgrid_post = Instant::now(); + + if false { + use std::fs::File; + let mut f = File::create(&format!( + "chonkjpegs/tmp_{}_{}.jpg", + spiralpos.x, spiralpos.y + )) + .unwrap(); + f.write_all(&*jpegchonkgrid).unwrap(); + } + + let jpegchonktall_pre = Instant::now(); + let jpegchonktall = + image_terrain_chonk(JpegEncoding, TallPacking { flip_y: false }, &chunk); + let jpegchonktall_post = Instant::now(); + + let jpegchonkflip_pre = Instant::now(); + let jpegchonkflip = + image_terrain_chonk(JpegEncoding, TallPacking { flip_y: true }, &chunk); + let jpegchonkflip_post = Instant::now(); + + let mixedchonk_pre = Instant::now(); + let mixedchonk = + image_terrain_chonk(MixedEncoding, TallPacking { flip_y: true }, &chunk); + let mixedchonk_post = Instant::now(); + + let mixeddeflate = CompressedData::compress(&mixedchonk, 1); + let mixeddeflate_post = Instant::now(); + + let mixeddense_pre = Instant::now(); + let mixeddense = + image_terrain_chonk(MixedEncodingDenseSprites, TallPacking { flip_y: true }, &chunk); + let mixeddense_post = Instant::now(); + + let pngchonk_pre = Instant::now(); + let pngchonk = image_terrain_chonk(PngEncoding, GridLtrPacking, &chunk); + let pngchonk_post = Instant::now(); + + let n = uncompressed.len(); + let sizes = [ + lz4_chonk.len() as f32 / n as f32, + deflate_chonk.len() as f32 / n as f32, + lz4_dyna.len() as f32 / n as f32, + deflate_dyna.len() as f32 / n as f32, + deflate_channeled_dyna.len() as f32 / n as f32, + jpegchonkgrid.len() as f32 / n as f32, + jpegchonktall.len() as f32 / n as f32, + jpegchonkflip.len() as f32 / n as f32, + mixedchonk.0.len() as f32 / n as f32, + mixeddeflate.data.len() as f32 / n as f32, + mixeddense.0.len() as f32 / n as f32, + pngchonk.len() as f32 / n as f32, + ]; + let best_idx = sizes + .iter() + .enumerate() + .fold((1.0, 0), |(best, i), (j, ratio)| { + if ratio < &best { + (*ratio, j) + } else { + (best, i) + } + }) + .1; + let timings = [ + (lz4chonk_post - lz4chonk_pre).subsec_nanos(), + (deflatechonk_post - deflatechonk_pre).subsec_nanos(), + (jpegchonkgrid_post - jpegchonkgrid_pre).subsec_nanos(), + (jpegchonktall_post - jpegchonktall_pre).subsec_nanos(), + (jpegchonkflip_post - jpegchonkflip_pre).subsec_nanos(), + (mixedchonk_post - mixedchonk_pre).subsec_nanos(), + (mixeddeflate_post - mixedchonk_pre).subsec_nanos(), + (mixeddense_post - mixeddense_pre).subsec_nanos(), + (pngchonk_post - pngchonk_pre).subsec_nanos(), + ]; + trace!( + "{} {}: uncompressed: {}, {:?} {} {:?}", + spiralpos.x, + spiralpos.y, + n, + sizes, + best_idx, + timings + ); + for j in 0..totals.len() { + totals[j] += sizes[j]; + } + for j in 0..total_timings.len() { + total_timings[j] += timings[j] as f32; + } + count += 1; + let _ = volgrid.insert(spiralpos, Arc::new(chunk)); + + if (1usize..20) + .into_iter() + .any(|i| (2 * i + 1) * (2 * i + 1) == count) + { + use std::fs::File; + let mut f = + File::create(&format!("chonkjpegs/{}_{}.jpg", sitename, count)).unwrap(); + let jpeg_volgrid = + image_terrain_volgrid(JpegEncoding, GridLtrPacking, &volgrid); + f.write_all(&*jpeg_volgrid).unwrap(); + + let mixedgrid_pre = Instant::now(); + let (mixed_volgrid, indices) = + image_terrain_volgrid(MixedEncoding, GridLtrPacking, &volgrid); + let mixedgrid_post = Instant::now(); + let seconds = (mixedgrid_post - mixedgrid_pre).as_secs_f64(); + println!( + "Generated mixed_volgrid in {} seconds for {} chunks ({} avg)", + seconds, + count, + seconds / count as f64, + ); + for i in 0..4 { + const FMT: [&str; 4] = ["png", "png", "png", "jpg"]; + let ranges: [_; 4] = [ + 0..indices[0], + indices[0]..indices[1], + indices[1]..indices[2], + indices[2]..mixed_volgrid.len(), + ]; + let mut f = File::create(&format!( + "chonkmixed/{}_{}_{}.{}", + sitename, count, i, FMT[i] + )) + .unwrap(); + f.write_all(&mixed_volgrid[ranges[i].clone()]).unwrap(); + } + } + } + if count % 64 == 0 { + println!("Chunks processed ({}): {}\n", sitename, count); + println!("Average lz4_chonk: {}", totals[0] / count as f32); + println!("Average deflate_chonk: {}", totals[1] / count as f32); + println!("Average lz4_dyna: {}", totals[2] / count as f32); + println!("Average deflate_dyna: {}", totals[3] / count as f32); + println!( + "Average deflate_channeled_dyna: {}", + totals[4] / count as f32 + ); + println!("Average jpeggridchonk: {}", totals[5] / count as f32); + println!("Average jpegtallchonk: {}", totals[6] / count as f32); + println!("Average jpegflipchonk: {}", totals[7] / count as f32); + println!("Average mixedchonk: {}", totals[8] / count as f32); + println!("Average mixeddeflate: {}", totals[9] / count as f32); + println!("Average mixeddense: {}", totals[10] / count as f32); + println!("Average pngchonk: {}", totals[11] / count as f32); + println!(""); + println!( + "Average lz4_chonk nanos : {:02}", + total_timings[0] / count as f32 + ); + println!( + "Average deflate_chonk nanos: {:02}", + total_timings[1] / count as f32 + ); + println!( + "Average jpeggridchonk nanos: {:02}", + total_timings[2] / count as f32 + ); + println!( + "Average jpegtallchonk nanos: {:02}", + total_timings[3] / count as f32 + ); + println!( + "Average jpegflipchonk nanos: {:02}", + total_timings[4] / count as f32 + ); + println!( + "Average mixedchonk nanos: {:02}", + total_timings[5] / count as f32 + ); + println!( + "Average mixeddeflate nanos: {:02}", + total_timings[6] / count as f32 + ); + println!( + "Average mixeddense nanos: {:02}", + total_timings[7] / count as f32 + ); + println!( + "Average pngchonk nanos: {:02}", + total_timings[8] / count as f32 + ); + println!("-----"); + } + if i % 256 == 0 { + histogram.clear(); } - } - if i % 64 == 0 { - println!("Chunks processed: {}\n", count); - println!("Average lz4_chonk: {}", totals[0] / count as f32); - println!("Average deflate_chonk: {}", totals[1] / count as f32); - println!("Average lz4_dyna: {}", totals[2] / count as f32); - println!("Average deflate_dyna: {}", totals[3] / count as f32); - println!( - "Average deflate_channeled_dyna: {}", - totals[4] / count as f32 - ); - println!("Average jpeggridchonk: {}", totals[5] / count as f32); - println!("Average jpegtallchonk: {}", totals[6] / count as f32); - println!("Average jpegflipchonk: {}", totals[7] / count as f32); - println!("Average mixedchonk: {}", totals[8] / count as f32); - println!("Average pngchonk: {}", totals[9] / count as f32); - println!(""); - println!( - "Average lz4_chonk nanos : {:02}", - total_timings[0] / count as f32 - ); - println!( - "Average deflate_chonk nanos: {:02}", - total_timings[1] / count as f32 - ); - println!( - "Average jpeggridchonk nanos: {:02}", - total_timings[2] / count as f32 - ); - println!( - "Average jpegtallchonk nanos: {:02}", - total_timings[3] / count as f32 - ); - println!( - "Average jpegflipchonk nanos: {:02}", - total_timings[4] / count as f32 - ); - println!( - "Average mixedchonk nanos: {:02}", - total_timings[5] / count as f32 - ); - println!( - "Average pngchonk nanos: {:02}", - total_timings[6] / count as f32 - ); - println!("-----"); - } - if i % 256 == 0 { - histogram.clear(); } } }