Optimize CompressedData with Vec::with_capacity, and move obsolete formats from common_net into the compression benchmark.

This commit is contained in:
Avi Weinstock 2021-04-28 19:26:49 -04:00
parent c199d12f2d
commit b15913560b
6 changed files with 248 additions and 237 deletions

1
Cargo.lock generated
View File

@ -5810,6 +5810,7 @@ dependencies = [
"minifb",
"noise",
"num 0.4.0",
"num-traits",
"ordered-float 2.1.1",
"packed_simd_2",
"rand 0.8.3",

View File

@ -34,7 +34,8 @@ impl<T: Serialize> CompressedData<T> {
const EXPECT_MSG: &str =
"compression only fails for fallible Read/Write impls (which Vec<u8> is not)";
let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(level));
let buf = Vec::with_capacity(uncompressed.len() / 10);
let mut encoder = DeflateEncoder::new(buf, Compression::new(level));
encoder.write_all(&*uncompressed).expect(EXPECT_MSG);
let compressed = encoder.finish().expect(EXPECT_MSG);
CompressedData {
@ -55,7 +56,7 @@ impl<T: Serialize> CompressedData<T> {
impl<T: for<'a> Deserialize<'a>> CompressedData<T> {
pub fn decompress(&self) -> Option<T> {
if self.compressed {
let mut uncompressed = Vec::new();
let mut uncompressed = Vec::with_capacity(self.data.len());
flate2::read::DeflateDecoder::new(&*self.data)
.read_to_end(&mut uncompressed)
.ok()?;
@ -72,36 +73,9 @@ pub trait PackingFormula: Copy {
fn index(&self, dims: Vec3<u32>, x: u32, y: u32, z: u32) -> (u32, u32);
}
/// A tall, thin image, with no wasted space, but which most image viewers don't
/// handle well. Z levels increase from top to bottom, xy-slices are stacked
/// vertically.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct TallPacking {
/// Making the borders go back and forth based on z-parity preserves spatial
/// locality better, but is more confusing to look at
pub flip_y: bool,
}
impl PackingFormula for TallPacking {
#[inline(always)]
fn dimensions(&self, dims: Vec3<u32>) -> (u32, u32) { (dims.x, dims.y * dims.z) }
#[allow(clippy::many_single_char_names)]
#[inline(always)]
fn index(&self, dims: Vec3<u32>, x: u32, y: u32, z: u32) -> (u32, u32) {
let i = x;
let j0 = if self.flip_y {
if z % 2 == 0 { y } else { dims.y - y - 1 }
} else {
y
};
let j = z * dims.y + j0;
(i, j)
}
}
/// A wide, short image. Shares the advantage of not wasting space with
/// TallPacking, but faster to compress and smaller since PNG compresses each
/// TallPacking (which is strictly worse, and was moved to benchmark code in
/// `world`), but faster to compress and smaller since PNG compresses each
/// row indepedently, so a wide image has fewer calls to the compressor. FLIP_X
/// has the same spatial locality preserving behavior as with TallPacking.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
@ -126,8 +100,9 @@ impl<const FLIP_X: bool> PackingFormula for WidePacking<FLIP_X> {
}
/// A grid of the z levels, left to right, top to bottom, like English prose.
/// Convenient for visualizing terrain, but wastes space if the number of z
/// levels isn't a perfect square.
/// Convenient for visualizing terrain for debugging or for user-inspectable
/// file formats, but wastes space if the number of z levels isn't a perfect
/// square.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct GridLtrPacking;
@ -169,160 +144,7 @@ pub trait VoxelImageDecoding: VoxelImageEncoding {
fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block;
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct PngEncoding;
impl VoxelImageEncoding for PngEncoding {
type Output = Vec<u8>;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
use image::Rgba;
ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
ori: Option<u8>,
) {
ws.put_pixel(
x,
y,
image::Rgba([kind as u8, sprite as u8, ori.unwrap_or(0), 255]),
);
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
use image::codecs::png::{CompressionType, FilterType};
let mut buf = Vec::new();
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.encode(
&*ws.as_raw(),
ws.width(),
ws.height(),
image::ColorType::Rgba8,
)
.ok()?;
Some(buf)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct JpegEncoding;
impl VoxelImageEncoding for JpegEncoding {
type Output = Vec<u8>;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
use image::Rgba;
ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
_: Option<u8>,
) {
ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
let mut buf = Vec::new();
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
jpeg.encode_image(ws).ok()?;
Some(buf)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct MixedEncoding;
impl VoxelImageEncoding for MixedEncoding {
type Output = (Vec<u8>, [usize; 3]);
#[allow(clippy::type_complexity)]
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Rgb<u8>, Vec<u8>>,
);
fn create(width: u32, height: u32) -> Self::Workspace {
(
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.0.put_pixel(x, y, image::Luma([kind as u8]));
ws.1.put_pixel(x, y, image::Luma([0]));
ws.2.put_pixel(x, y, image::Luma([0]));
ws.3.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
ori: Option<u8>,
) {
ws.0.put_pixel(x, y, image::Luma([kind as u8]));
ws.1.put_pixel(x, y, image::Luma([sprite as u8]));
ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
ws.3.put_pixel(x, y, image::Rgb([0; 3]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.encode(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
indices[i] = buf.len();
Some(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
f(&ws.2, 2)?;
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 10);
jpeg.encode_image(&ws.3).ok()?;
Some((buf, indices))
}
}
fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u8>>(
pub fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u8>>(
decoder: I,
) -> Option<ImageBuffer<P, Vec<u8>>> {
let (w, h) = decoder.dimensions();
@ -331,47 +153,6 @@ fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u8>>(
ImageBuffer::from_raw(w, h, buf)
}
impl VoxelImageDecoding for MixedEncoding {
fn start((quad, indices): &Self::Output) -> Option<Self::Workspace> {
use image::codecs::{jpeg::JpegDecoder, png::PngDecoder};
let ranges: [_; 4] = [
0..indices[0],
indices[0]..indices[1],
indices[1]..indices[2],
indices[2]..quad.len(),
];
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
Some((a, b, c, d))
}
fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
if let Some(kind) = BlockKind::from_u8(ws.0.get_pixel(x, y).0[0]) {
if kind.is_filled() {
let rgb = ws.3.get_pixel(x, y);
Block::new(kind, Rgb {
r: rgb[0],
g: rgb[1],
b: rgb[2],
})
} else {
let mut block = Block::new(kind, Rgb { r: 0, g: 0, b: 0 });
if let Some(spritekind) = SpriteKind::from_u8(ws.1.get_pixel(x, y).0[0]) {
block = block.with_sprite(spritekind);
}
if let Some(oriblock) = block.with_ori(ws.2.get_pixel(x, y).0[0]) {
block = oriblock;
}
block
}
} else {
Block::empty()
}
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct QuadPngEncoding<const RESOLUTION_DIVIDER: u32>();

View File

@ -8,8 +8,8 @@ pub mod world_msg;
pub use self::{
client::{ClientGeneral, ClientMsg, ClientRegister, ClientType},
compression::{
CompressedData, GridLtrPacking, JpegEncoding, MixedEncoding, PackingFormula, PngEncoding,
QuadPngEncoding, TallPacking, TriPngEncoding, VoxelImageEncoding, WidePacking, WireChonk,
CompressedData, GridLtrPacking, PackingFormula, QuadPngEncoding, TriPngEncoding,
VoxelImageEncoding, WidePacking, WireChonk,
},
ecs_packet::EcsCompPacket,
server::{

View File

@ -21,21 +21,25 @@ use specs::{Join, Read, ReadExpect, ReadStorage, Write, WriteExpect};
use std::sync::Arc;
use vek::*;
pub struct LazyTerrainMessage {
pub(crate) struct LazyTerrainMessage {
lazy_msg_lo: Option<crate::client::PreparedMsg>,
lazy_msg_hi: Option<crate::client::PreparedMsg>,
}
impl LazyTerrainMessage {
#[allow(clippy::new_without_default)]
pub fn new() -> Self {
pub(crate) fn new() -> Self {
Self {
lazy_msg_lo: None,
lazy_msg_hi: None,
}
}
pub fn prepare_and_send<'a, A, F: FnOnce() -> Result<&'a common::terrain::TerrainChunk, A>>(
pub(crate) fn prepare_and_send<
'a,
A,
F: FnOnce() -> Result<&'a common::terrain::TerrainChunk, A>,
>(
&mut self,
network_metrics: &NetworkRequestMetrics,
client: &Client,

View File

@ -7,7 +7,7 @@ edition = "2018"
[features]
tracy = ["common/tracy", "common-net/tracy"]
simd = ["vek/platform_intrinsics"]
bin_compression = ["lz-fear", "deflate", "flate2", "common-frontend", "image/jpeg"]
bin_compression = ["lz-fear", "deflate", "flate2", "common-frontend", "image/jpeg", "num-traits"]
default = ["simd"]
@ -42,6 +42,7 @@ assets_manager = {version = "0.4.3", features = ["ron"]}
lz-fear = { version = "0.1.1", optional = true }
deflate = { version = "0.9.1", optional = true }
flate2 = { version = "1.0.20", optional = true }
num-traits = { version = "0.2", optional = true }
common-frontend = { package = "veloren-common-frontend", path = "../common/frontend", optional = true }

View File

@ -8,12 +8,14 @@ use common::{
},
};
use common_net::msg::compression::{
image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking, JpegEncoding,
MixedEncoding, PngEncoding, QuadPngEncoding, TallPacking, TriPngEncoding, VoxelImageEncoding,
image_from_bytes, image_terrain_chonk, image_terrain_volgrid, CompressedData, GridLtrPacking,
PackingFormula, QuadPngEncoding, TriPngEncoding, VoxelImageDecoding, VoxelImageEncoding,
WidePacking,
};
use hashbrown::HashMap;
use image::ImageBuffer;
use num_traits::cast::FromPrimitive;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
io::{Read, Write},
@ -132,6 +134,228 @@ fn channelize_dyna<M: Clone, A: Access>(
(blocks, r, g, b, sprites)
}
/// A tall, thin image, with no wasted space, but which most image viewers don't
/// handle well. Z levels increase from top to bottom, xy-slices are stacked
/// vertically.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct TallPacking {
/// Making the borders go back and forth based on z-parity preserves spatial
/// locality better, but is more confusing to look at
pub flip_y: bool,
}
impl PackingFormula for TallPacking {
#[inline(always)]
fn dimensions(&self, dims: Vec3<u32>) -> (u32, u32) { (dims.x, dims.y * dims.z) }
#[allow(clippy::many_single_char_names)]
#[inline(always)]
fn index(&self, dims: Vec3<u32>, x: u32, y: u32, z: u32) -> (u32, u32) {
let i = x;
let j0 = if self.flip_y {
if z % 2 == 0 { y } else { dims.y - y - 1 }
} else {
y
};
let j = z * dims.y + j0;
(i, j)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct PngEncoding;
impl VoxelImageEncoding for PngEncoding {
type Output = Vec<u8>;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
use image::Rgba;
ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
ori: Option<u8>,
) {
ws.put_pixel(
x,
y,
image::Rgba([kind as u8, sprite as u8, ori.unwrap_or(0), 255]),
);
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
use image::codecs::png::{CompressionType, FilterType};
let mut buf = Vec::new();
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.encode(
&*ws.as_raw(),
ws.width(),
ws.height(),
image::ColorType::Rgba8,
)
.ok()?;
Some(buf)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct JpegEncoding;
impl VoxelImageEncoding for JpegEncoding {
type Output = Vec<u8>;
type Workspace = ImageBuffer<image::Rgba<u8>, Vec<u8>>;
fn create(width: u32, height: u32) -> Self::Workspace {
use image::Rgba;
ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
_: Option<u8>,
) {
ws.put_pixel(x, y, image::Rgba([kind as u8, sprite as u8, 255, 255]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
let mut buf = Vec::new();
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 1);
jpeg.encode_image(ws).ok()?;
Some(buf)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct MixedEncoding;
impl VoxelImageEncoding for MixedEncoding {
type Output = (Vec<u8>, [usize; 3]);
#[allow(clippy::type_complexity)]
type Workspace = (
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Luma<u8>, Vec<u8>>,
ImageBuffer<image::Rgb<u8>, Vec<u8>>,
);
fn create(width: u32, height: u32) -> Self::Workspace {
(
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
ImageBuffer::new(width, height),
)
}
fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
ws.0.put_pixel(x, y, image::Luma([kind as u8]));
ws.1.put_pixel(x, y, image::Luma([0]));
ws.2.put_pixel(x, y, image::Luma([0]));
ws.3.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b]));
}
fn put_sprite(
ws: &mut Self::Workspace,
x: u32,
y: u32,
kind: BlockKind,
sprite: SpriteKind,
ori: Option<u8>,
) {
ws.0.put_pixel(x, y, image::Luma([kind as u8]));
ws.1.put_pixel(x, y, image::Luma([sprite as u8]));
ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
ws.3.put_pixel(x, y, image::Rgb([0; 3]));
}
fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
let mut buf = Vec::new();
use image::codecs::png::{CompressionType, FilterType};
let mut indices = [0; 3];
let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
let png = image::codecs::png::PngEncoder::new_with_quality(
&mut buf,
CompressionType::Rle,
FilterType::Up,
);
png.encode(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
.ok()?;
indices[i] = buf.len();
Some(())
};
f(&ws.0, 0)?;
f(&ws.1, 1)?;
f(&ws.2, 2)?;
let mut jpeg = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 10);
jpeg.encode_image(&ws.3).ok()?;
Some((buf, indices))
}
}
impl VoxelImageDecoding for MixedEncoding {
fn start((quad, indices): &Self::Output) -> Option<Self::Workspace> {
use image::codecs::{jpeg::JpegDecoder, png::PngDecoder};
let ranges: [_; 4] = [
0..indices[0],
indices[0]..indices[1],
indices[1]..indices[2],
indices[2]..quad.len(),
];
let a = image_from_bytes(PngDecoder::new(&quad[ranges[0].clone()]).ok()?)?;
let b = image_from_bytes(PngDecoder::new(&quad[ranges[1].clone()]).ok()?)?;
let c = image_from_bytes(PngDecoder::new(&quad[ranges[2].clone()]).ok()?)?;
let d = image_from_bytes(JpegDecoder::new(&quad[ranges[3].clone()]).ok()?)?;
Some((a, b, c, d))
}
fn get_block(ws: &Self::Workspace, x: u32, y: u32, _: bool) -> Block {
if let Some(kind) = BlockKind::from_u8(ws.0.get_pixel(x, y).0[0]) {
if kind.is_filled() {
let rgb = ws.3.get_pixel(x, y);
Block::new(kind, Rgb {
r: rgb[0],
g: rgb[1],
b: rgb[2],
})
} else {
let mut block = Block::new(kind, Rgb { r: 0, g: 0, b: 0 });
if let Some(spritekind) = SpriteKind::from_u8(ws.1.get_pixel(x, y).0[0]) {
block = block.with_sprite(spritekind);
}
if let Some(oriblock) = block.with_ori(ws.2.get_pixel(x, y).0[0]) {
block = oriblock;
}
block
}
} else {
Block::empty()
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct MixedEncodingSparseSprites;
@ -506,7 +730,7 @@ fn main() {
]);
if HISTOGRAMS {
let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2);
sizes.push(("lz4_dict_dyna", lz4_dyna.len() as f32 / n as f32));
sizes.push(("lz4_dict_dyna", lz4_dict_dyna.len() as f32 / n as f32));
}
}