From 4517faae9db3e053bee3afb70b20477cd6f274b7 Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Sat, 24 Apr 2021 14:02:32 -0400 Subject: [PATCH] Experiment with 256-color palette "tripng" encoding, and Lanczos interpolation for "quadpng". --- Cargo.lock | 1 + common/net/Cargo.toml | 2 +- common/net/src/msg/compression.rs | 208 +++++++++++++++++- common/net/src/msg/mod.rs | 2 +- common/net/src/msg/server.rs | 27 ++- world/src/bin/chunk_compression_benchmarks.rs | 23 +- 6 files changed, 247 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66f8e2da55..7f6464b0af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5719,6 +5719,7 @@ dependencies = [ "flate2", "hashbrown", "image", + "inline_tweak", "num-traits", "serde", "specs", diff --git a/common/net/Cargo.toml b/common/net/Cargo.toml index 4f7370f6a2..099aca7835 100644 --- a/common/net/Cargo.toml +++ b/common/net/Cargo.toml @@ -21,7 +21,7 @@ num-traits = "0.2" sum_type = "0.2.0" vek = { version = "=0.14.1", features = ["serde"] } tracing = { version = "0.1", default-features = false } -#inline_tweak = "1.0.2" +inline_tweak = "1.0.2" # Data structures hashbrown = { version = "0.9", features = ["rayon", "serde", "nightly"] } diff --git a/common/net/src/msg/compression.rs b/common/net/src/msg/compression.rs index d5f1f81068..24a062ed44 100644 --- a/common/net/src/msg/compression.rs +++ b/common/net/src/msg/compression.rs @@ -3,6 +3,7 @@ use common::{ vol::{BaseVol, ReadVol, RectVolSize, WriteVol}, volumes::vol_grid_2d::VolGrid2d, }; +use hashbrown::HashMap; use image::{ImageBuffer, ImageDecoder, Pixel}; use num_traits::cast::FromPrimitive; use serde::{Deserialize, Serialize}; @@ -428,6 +429,18 @@ impl VoxelImageEncoding for QuadPngEncoding { } } +/// https://en.wikipedia.org/wiki/Lanczos_resampling#Lanczos_kernel +fn lanczos(x: f64, a: f64) -> f64 { + use std::f64::consts::PI; + if x < f64::EPSILON { + 1.0 + } else if -a <= x && x <= a { + (a * (PI * x).sin() * (PI * x / a).sin()) / (PI.powi(2) * x.powi(2)) + } else { + 0.0 + } +} + impl VoxelImageDecoding for QuadPngEncoding { fn start(data: &Self::Output) -> Option { use image::codecs::png::PngDecoder; @@ -445,15 +458,198 @@ impl VoxelImageDecoding for QuadPngEncoding { Some((a, b, c, d)) } + #[allow(clippy::many_single_char_names)] + fn get_block(ws: &Self::Workspace, x: u32, y: u32) -> Block { + //let a = inline_tweak::tweak!(1.3); + //let b = inline_tweak::tweak!(4.0); + let a = 1.3; + let b = 4.0; + if let Some(kind) = BlockKind::from_u8(ws.0.get_pixel(x, y).0[0]) { + if kind.is_filled() { + let (w, h) = ws.3.dimensions(); + let rgb = match 1 { + 0 => { + let mut rgb: Vec3 = + Vec3::::from(ws.3.get_pixel(x / N, y / N).0).as_(); + rgb *= 2.0; + let mut total = 2; + for (dx, dy) in [(-1i32, 0i32), (1, 0), (0, -1), (0, 1)].iter() { + let (i, j) = ( + (x / N).wrapping_add(*dx as u32), + (y / N).wrapping_add(*dy as u32), + ); + if i < w && j < h { + rgb += Vec3::::from(ws.3.get_pixel(i, j).0).as_(); + total += 1; + } + } + rgb /= total as f64; + rgb + }, + _ => { + let mut rgb: Vec3 = Vec3::zero(); + for dx in -1i32..=1 { + for dy in -1i32..=1 { + let (i, j) = ( + (x / N).wrapping_add(dx as u32), + (y / N).wrapping_add(dy as u32), + ); + if i < w && j < h { + let pix: Vec3 = + Vec3::::from(ws.3.get_pixel(i, j).0).as_(); + rgb += lanczos( + a * Vec2::new( + (x % N) as f64 - (N - 1) as f64 / 2.0, + (y % N) as f64 - (N - 1) as f64 / 2.0, + ) + .magnitude(), + b, + ) * pix; + } + } + } + rgb + }, + }; + //let rgb = ; + Block::new(kind, Rgb { + r: rgb.x as u8, + g: rgb.y as u8, + b: rgb.z as u8, + }) + } else { + let mut block = Block::new(kind, Rgb { r: 0, g: 0, b: 0 }); + if let Some(spritekind) = SpriteKind::from_u8(ws.1.get_pixel(x, y).0[0]) { + block = block.with_sprite(spritekind); + } + if let Some(oriblock) = block.with_ori(ws.2.get_pixel(x, y).0[0]) { + block = oriblock; + } + block + } + } else { + Block::empty() + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct TriPngEncoding; + +impl VoxelImageEncoding for TriPngEncoding { + type Output = CompressedData<(Vec, Vec>, [usize; 3])>; + #[allow(clippy::type_complexity)] + type Workspace = ( + ImageBuffer, Vec>, + ImageBuffer, Vec>, + ImageBuffer, Vec>, + HashMap, usize>>, + ); + + fn create(width: u32, height: u32) -> Self::Workspace { + ( + ImageBuffer::new(width, height), + ImageBuffer::new(width, height), + ImageBuffer::new(width, height), + HashMap::new(), + ) + } + + fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.1.put_pixel(x, y, image::Luma([0])); + ws.2.put_pixel(x, y, image::Luma([0])); + *ws.3.entry(kind).or_default().entry(rgb).or_insert(0) += 1; + } + + fn put_sprite( + ws: &mut Self::Workspace, + x: u32, + y: u32, + kind: BlockKind, + sprite: SpriteKind, + ori: Option, + ) { + ws.0.put_pixel(x, y, image::Luma([kind as u8])); + ws.1.put_pixel(x, y, image::Luma([sprite as u8])); + ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)])); + } + + fn finish(ws: &Self::Workspace) -> Option { + let mut buf = Vec::new(); + use image::codecs::png::{CompressionType, FilterType}; + let mut indices = [0; 3]; + let mut f = |x: &ImageBuffer<_, Vec>, i| { + let png = image::codecs::png::PngEncoder::new_with_quality( + &mut buf, + CompressionType::Fast, + FilterType::Up, + ); + png.encode(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8) + .ok()?; + indices[i] = buf.len(); + Some(()) + }; + f(&ws.0, 0)?; + f(&ws.1, 1)?; + f(&ws.2, 2)?; + + let mut palette = vec![Rgb { r: 0, g: 0, b: 0 }; 256]; + for (block, hist) in ws.3.iter() { + let (mut r, mut g, mut b) = (0.0, 0.0, 0.0); + let mut total = 0; + for (color, count) in hist.iter() { + r += color.r as f64 * *count as f64; + g += color.g as f64 * *count as f64; + b += color.b as f64 * *count as f64; + total += *count; + } + r /= total as f64; + g /= total as f64; + b /= total as f64; + palette[*block as u8 as usize].r = r as u8; + palette[*block as u8 as usize].g = g as u8; + palette[*block as u8 as usize].b = b as u8; + } + + Some(CompressedData::compress(&(buf, palette, indices), 4)) + } +} + +impl VoxelImageDecoding for TriPngEncoding { + fn start(data: &Self::Output) -> Option { + use image::codecs::png::PngDecoder; + let (quad, palette, indices) = data.decompress()?; + let ranges: [_; 3] = [ + 0..indices[0], + indices[0]..indices[1], + indices[1]..indices[2], + ]; + let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?; + let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?; + let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?; + let mut d: HashMap<_, HashMap<_, _>> = HashMap::new(); + for i in 0..=255 { + if let Some(block) = BlockKind::from_u8(i) { + d.entry(block) + .or_default() + .entry(palette[i as usize]) + .insert(1); + } + } + + Some((a, b, c, d)) + } + fn get_block(ws: &Self::Workspace, x: u32, y: u32) -> Block { if let Some(kind) = BlockKind::from_u8(ws.0.get_pixel(x, y).0[0]) { if kind.is_filled() { - let rgb = ws.3.get_pixel(x / N, y / N); - Block::new(kind, Rgb { - r: rgb[0], - g: rgb[1], - b: rgb[2], - }) + let rgb = *ws + .3 + .get(&kind) + .and_then(|h| h.keys().next()) + .unwrap_or(&Rgb::default()); + Block::new(kind, rgb) } else { let mut block = Block::new(kind, Rgb { r: 0, g: 0, b: 0 }); if let Some(spritekind) = SpriteKind::from_u8(ws.1.get_pixel(x, y).0[0]) { diff --git a/common/net/src/msg/mod.rs b/common/net/src/msg/mod.rs index 446c929ad5..58ceecee9b 100644 --- a/common/net/src/msg/mod.rs +++ b/common/net/src/msg/mod.rs @@ -9,7 +9,7 @@ pub use self::{ client::{ClientGeneral, ClientMsg, ClientRegister, ClientType}, compression::{ CompressedData, GridLtrPacking, JpegEncoding, MixedEncoding, PackingFormula, PngEncoding, - QuadPngEncoding, TallPacking, VoxelImageEncoding, WireChonk, + QuadPngEncoding, TallPacking, TriPngEncoding, VoxelImageEncoding, WireChonk, }, ecs_packet::EcsCompPacket, server::{ diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs index 5f27954016..c8864b3129 100644 --- a/common/net/src/msg/server.rs +++ b/common/net/src/msg/server.rs @@ -1,6 +1,6 @@ use super::{ world_msg::EconomyInfo, ClientType, CompressedData, EcsCompPacket, MixedEncoding, PingMsg, - QuadPngEncoding, TallPacking, WireChonk, + QuadPngEncoding, TallPacking, TriPngEncoding, WireChonk, }; use crate::sync; use common::{ @@ -70,10 +70,20 @@ pub type ServerRegisterAnswer = Result<(), RegisterError>; pub enum SerializedTerrainChunk { DeflatedChonk(CompressedData), PngPngPngJpeg(WireChonk), - QuadPng(WireChonk, TallPacking, TerrainChunkMeta, TerrainChunkSize>), + QuadPng(WireChonk, TallPacking, TerrainChunkMeta, TerrainChunkSize>), + TriPng(WireChonk), } impl SerializedTerrainChunk { + pub fn image(chunk: &TerrainChunk) -> Self { + match inline_tweak::tweak!(2) { + 0 => Self::deflate(chunk), + 1 => Self::jpeg(chunk), + 2 => Self::quadpng(chunk), + _ => Self::tripng(chunk), + } + } + pub fn deflate(chunk: &TerrainChunk) -> Self { Self::DeflatedChonk(CompressedData::compress(chunk, 5)) } @@ -88,7 +98,7 @@ impl SerializedTerrainChunk { } } - pub fn image(chunk: &TerrainChunk) -> Self { + pub fn quadpng(chunk: &TerrainChunk) -> Self { if let Some(wc) = WireChonk::from_chonk(QuadPngEncoding(), TallPacking { flip_y: true }, chunk) { @@ -99,11 +109,22 @@ impl SerializedTerrainChunk { } } + pub fn tripng(chunk: &TerrainChunk) -> Self { + if let Some(wc) = WireChonk::from_chonk(TriPngEncoding, TallPacking { flip_y: true }, chunk) + { + Self::TriPng(wc) + } else { + warn!("Image encoding failure occurred, falling back to deflate"); + Self::deflate(chunk) + } + } + pub fn to_chunk(&self) -> Option { match self { Self::DeflatedChonk(chonk) => chonk.decompress(), Self::PngPngPngJpeg(wc) => wc.to_chonk(), Self::QuadPng(wc) => wc.to_chonk(), + Self::TriPng(wc) => wc.to_chonk(), } } } diff --git a/world/src/bin/chunk_compression_benchmarks.rs b/world/src/bin/chunk_compression_benchmarks.rs index 3f1ed380c1..ee66bbd702 100644 --- a/world/src/bin/chunk_compression_benchmarks.rs +++ b/world/src/bin/chunk_compression_benchmarks.rs @@ -9,7 +9,7 @@ use common::{ }; use common_net::msg::compression::{ image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking, JpegEncoding, - MixedEncoding, PngEncoding, QuadPngEncoding, TallPacking, VoxelImageEncoding, + MixedEncoding, PngEncoding, QuadPngEncoding, TallPacking, TriPngEncoding, VoxelImageEncoding, }; use hashbrown::HashMap; use image::ImageBuffer; @@ -328,8 +328,8 @@ fn main() { )); for (sitename, sitepos) in sites.iter() { - let mut totals = [0.0; 15]; - let mut total_timings = [0.0; 12]; + let mut totals = [0.0; 16]; + let mut total_timings = [0.0; 13]; let mut count = 0; let mut volgrid = VolGrid2d::new().unwrap(); for (i, spiralpos) in Spiral2d::new() @@ -446,6 +446,12 @@ fn main() { .unwrap(); let quadpngquart_post = Instant::now(); + let tripng_pre = Instant::now(); + let tripng = + image_terrain_chonk(TriPngEncoding, TallPacking { flip_y: true }, &chunk) + .unwrap(); + let tripng_post = Instant::now(); + let pngchonk_pre = Instant::now(); let pngchonk = image_terrain_chonk(PngEncoding, GridLtrPacking, &chunk).unwrap(); let pngchonk_post = Instant::now(); @@ -466,6 +472,7 @@ fn main() { quadpngfull.data.len() as f32 / n as f32, quadpnghalf.data.len() as f32 / n as f32, quadpngquart.data.len() as f32 / n as f32, + tripng.data.len() as f32 / n as f32, pngchonk.len() as f32 / n as f32, ]; let best_idx = sizes @@ -491,6 +498,7 @@ fn main() { (quadpngfull_post - quadpngfull_pre).subsec_nanos(), (quadpnghalf_post - quadpnghalf_pre).subsec_nanos(), (quadpngquart_post - quadpngquart_pre).subsec_nanos(), + (tripng_post - tripng_pre).subsec_nanos(), (pngchonk_post - pngchonk_pre).subsec_nanos(), ]; trace!( @@ -569,7 +577,8 @@ fn main() { println!("Average quadpngfull: {}", totals[11] / count as f32); println!("Average quadpnghalf: {}", totals[12] / count as f32); println!("Average quadpngquart: {}", totals[13] / count as f32); - println!("Average pngchonk: {}", totals[14] / count as f32); + println!("Average tripng: {}", totals[14] / count as f32); + println!("Average pngchonk: {}", totals[15] / count as f32); println!(""); println!( "Average lz4_chonk nanos : {:02}", @@ -616,9 +625,13 @@ fn main() { total_timings[10] / count as f32 ); println!( - "Average pngchonk nanos: {:02}", + "Average tripng nanos: {:02}", total_timings[11] / count as f32 ); + println!( + "Average pngchonk nanos: {:02}", + total_timings[12] / count as f32 + ); println!("-----"); } if i % 256 == 0 {