Merge branch 'aweinstock/blockstats' into 'master'

Add block statistics generator to world/examples.

See merge request veloren/veloren!2526

Cargo.lock

@@ -106,7 +106,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01aa95c33b750236bd69526dd773d618c5c8ad5e156daad1fc760dfd4be8def3"
 dependencies = [
- "heapless",
+ "heapless 0.5.6",
  "nom 4.2.3",
 ]
 
@@ -2269,6 +2269,18 @@ dependencies = [
  "stable_deref_trait",
 ]
 
+[[package]]
+name = "heapless"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422"
+dependencies = [
+ "as-slice",
+ "generic-array 0.14.4",
+ "hash32",
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -2658,6 +2670,15 @@ dependencies = [
  "libloading 0.7.0",
 ]
 
+[[package]]
+name = "kiddo"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73e70400c6435a3f5e4d7bfb406af50bc66950cc8f24f45488d66f47bf3f5e25"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "lazy-bytes-cast"
 version = "5.0.1"
@@ -3681,6 +3702,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "pdqselect"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27"
+
 [[package]]
 name = "peeking_take_while"
 version = "0.1.2"
@@ -4365,6 +4392,18 @@ dependencies = [
  "petgraph 0.5.1",
 ]
 
+[[package]]
+name = "rstar"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce61d743ebe516592df4dd542dfe823577b811299f7bee1106feb1bbb993dbac"
+dependencies = [
+ "heapless 0.6.1",
+ "num-traits",
+ "pdqselect",
+ "smallvec",
+]
+
 [[package]]
 name = "rusqlite"
 version = "0.24.2"
@@ -6134,15 +6173,18 @@ dependencies = [
  "arr_macro",
  "bincode",
  "bitvec",
+ "clap",
  "criterion",
  "csv",
  "deflate 0.9.1",
  "enum-iterator",
+ "fallible-iterator",
  "flate2",
  "fxhash",
  "hashbrown 0.11.2",
  "image",
  "itertools 0.10.0",
+ "kiddo",
  "lazy_static",
  "lz-fear",
  "minifb",
@@ -6155,6 +6197,7 @@ dependencies = [
  "rand_chacha 0.3.0",
  "rayon",
  "ron",
+ "rstar",
  "rusqlite",
  "serde",
  "structopt",

common/net/src/msg/compression.rs

@@ -123,12 +123,13 @@ impl PackingFormula for GridLtrPacking {
     }
 }
 
-pub trait VoxelImageEncoding: Copy {
+pub trait VoxelImageEncoding {
     type Workspace;
     type Output;
     fn create(width: u32, height: u32) -> Self::Workspace;
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>);
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>);
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -153,6 +154,37 @@ pub fn image_from_bytes<'a, I: ImageDecoder<'a>, P: 'static + Pixel<Subpixel = u
     ImageBuffer::from_raw(w, h, buf)
 }
 
+impl<'a, VIE: VoxelImageEncoding> VoxelImageEncoding for &'a VIE {
+    type Output = VIE::Output;
+    type Workspace = VIE::Workspace;
+
+    fn create(width: u32, height: u32) -> Self::Workspace { VIE::create(width, height) }
+
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+        (*self).put_solid(ws, x, y, kind, rgb)
+    }
+
+    fn put_sprite(
+        &self,
+        ws: &mut Self::Workspace,
+        x: u32,
+        y: u32,
+        kind: BlockKind,
+        sprite: SpriteKind,
+        ori: Option<u8>,
+    ) {
+        (*self).put_sprite(ws, x, y, kind, sprite, ori)
+    }
+
+    fn finish(ws: &Self::Workspace) -> Option<Self::Output> { VIE::finish(ws) }
+}
+impl<'a, VIE: VoxelImageDecoding> VoxelImageDecoding for &'a VIE {
+    fn start(ws: &Self::Output) -> Option<Self::Workspace> { VIE::start(ws) }
+
+    fn get_block(ws: &Self::Workspace, x: u32, y: u32, is_border: bool) -> Block {
+        VIE::get_block(ws, x, y, is_border)
+    }
+}
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub struct QuadPngEncoding<const RESOLUTION_DIVIDER: u32>();
 
@@ -176,13 +208,14 @@ impl<const N: u32> VoxelImageEncoding for QuadPngEncoding<N> {
     }
 
     #[inline(always)]
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.0.put_pixel(x, y, image::Luma([kind as u8]));
         ws.3.put_pixel(x / N, y / N, image::Rgb([rgb.r, rgb.g, rgb.b]));
     }
 
     #[inline(always)]
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -451,7 +484,7 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<AVERAGE_
         )
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.0.put_pixel(x, y, image::Luma([kind as u8]));
         ws.1.put_pixel(x, y, image::Luma([0]));
         ws.2.put_pixel(x, y, image::Luma([0]));
@@ -461,6 +494,7 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageEncoding for TriPngEncoding<AVERAGE_
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -623,7 +657,7 @@ impl<const AVERAGE_PALETTE: bool> VoxelImageDecoding for TriPngEncoding<AVERAGE_
 }
 
 pub fn image_terrain_chonk<S: RectVolSize, M: Clone, P: PackingFormula, VIE: VoxelImageEncoding>(
-    vie: VIE,
+    vie: &VIE,
     packing: P,
     chonk: &Chonk<Block, S, M>,
 ) -> Option<VIE::Output> {
@@ -642,7 +676,7 @@ pub fn image_terrain_volgrid<
     P: PackingFormula,
     VIE: VoxelImageEncoding,
 >(
-    vie: VIE,
+    vie: &VIE,
     packing: P,
     volgrid: &VolGrid2d<Chonk<Block, S, M>>,
 ) -> Option<VIE::Output> {
@@ -666,7 +700,7 @@ pub fn image_terrain<
     P: PackingFormula,
     VIE: VoxelImageEncoding,
 >(
-    _: VIE,
+    vie: &VIE,
     packing: P,
     vol: &V,
     lo: Vec3<u32>,
@@ -697,10 +731,10 @@ pub fn image_terrain<
                     .unwrap_or(&Block::empty());
                 match (block.get_color(), block.get_sprite()) {
                     (Some(rgb), None) => {
-                        VIE::put_solid(&mut image, i, j, *block, rgb);
+                        VIE::put_solid(vie, &mut image, i, j, *block, rgb);
                     },
                     (None, Some(sprite)) => {
-                        VIE::put_sprite(&mut image, i, j, *block, sprite, block.get_ori());
+                        VIE::put_sprite(vie, &mut image, i, j, *block, sprite, block.get_ori());
                     },
                     _ => panic!(
                         "attr being used for color vs sprite is mutually exclusive (and that's \
@@ -769,7 +803,7 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
     WireChonk<VIE, P, M, S>
 {
     pub fn from_chonk(vie: VIE, packing: P, chonk: &Chonk<Block, S, M>) -> Option<Self> {
-        let data = image_terrain_chonk(vie, packing, chonk)?;
+        let data = image_terrain_chonk(&vie, packing, chonk)?;
         Some(Self {
             zmin: chonk.get_min_z(),
             zmax: chonk.get_max_z(),
@@ -788,7 +822,7 @@ impl<VIE: VoxelImageEncoding + VoxelImageDecoding, P: PackingFormula, M: Clone,
     pub fn to_chonk(&self) -> Option<Chonk<Block, S, M>> {
         let mut chonk = Chonk::new(self.zmin, self.below, self.above, self.meta.clone());
         write_image_terrain(
-            self.vie,
+            &self.vie,
             self.packing,
             &mut chonk,
             &self.data,

world/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2018"
 
 [features]
 simd = ["vek/platform_intrinsics"]
-bin_compression = ["lz-fear", "deflate", "flate2", "image/jpeg", "num-traits"]
+bin_compression = ["lz-fear", "deflate", "flate2", "image/jpeg", "num-traits", "fallible-iterator", "kiddo", "clap", "rstar"]
 
 default = ["simd"]
 
@@ -41,6 +41,10 @@ lz-fear = { version = "0.1.1", optional = true }
 deflate = { version = "0.9.1", optional = true }
 flate2 = { version = "1.0.20", optional = true }
 num-traits = { version = "0.2", optional = true }
+fallible-iterator = { version = "0.2.0", optional = true }
+kiddo = { version = "0.1.4", optional = true }
+rstar = { version = "0.8.3", optional = true }
+clap = { version = "2.33.3", optional = true }
 
 
 [dev-dependencies]
@@ -62,6 +66,10 @@ name = "tree"
 name = "chunk_compression_benchmarks"
 required-features = ["bin_compression"]
 
+[[example]]
+name = "world_block_statistics"
+required-features = ["bin_compression"]
+
 [[example]]
 name = "heightmap_visualization"
 required-features = ["bin_compression"]

world/examples/chunk_compression_benchmarks.rs

@@ -173,11 +173,12 @@ impl VoxelImageEncoding for PngEncoding {
         ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -223,11 +224,12 @@ impl VoxelImageEncoding for JpegEncoding {
         ImageBuffer::<Rgba<u8>, Vec<u8>>::new(width, height)
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.put_pixel(x, y, image::Rgba([rgb.r, rgb.g, rgb.b, 255 - kind as u8]));
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -268,7 +270,7 @@ impl VoxelImageEncoding for MixedEncoding {
         )
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.0.put_pixel(x, y, image::Luma([kind as u8]));
         ws.1.put_pixel(x, y, image::Luma([0]));
         ws.2.put_pixel(x, y, image::Luma([0]));
@@ -276,6 +278,7 @@ impl VoxelImageEncoding for MixedEncoding {
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -378,12 +381,13 @@ impl VoxelImageEncoding for MixedEncodingSparseSprites {
         )
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.0.put_pixel(x, y, image::Luma([kind as u8]));
         ws.1.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b]));
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -439,12 +443,13 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
         )
     }
 
-    fn put_solid(ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
         ws.0.put_pixel(x, y, image::Luma([kind as u8]));
         ws.3.put_pixel(x, y, image::Rgb([rgb.r, rgb.g, rgb.b]));
     }
 
     fn put_sprite(
+        &self,
         ws: &mut Self::Workspace,
         x: u32,
         y: u32,
@@ -488,6 +493,168 @@ impl VoxelImageEncoding for MixedEncodingDenseSprites {
     }
 }
 
+use kiddo::KdTree;
+use rstar::{PointDistance, RTree, RTreeObject, RTreeParams};
+
+#[derive(Debug)]
+struct ColorPoint {
+    rgb: Rgb<u8>,
+    index: u8,
+}
+
+impl RTreeObject for ColorPoint {
+    type Envelope = <[i32; 3] as RTreeObject>::Envelope;
+
+    fn envelope(&self) -> Self::Envelope {
+        [self.rgb.r as i32, self.rgb.g as i32, self.rgb.b as i32].envelope()
+    }
+}
+
+impl PointDistance for ColorPoint {
+    fn distance_2(&self, other: &[i32; 3]) -> i32 {
+        (self.rgb.r as i32 - other[0]).pow(2)
+            + (self.rgb.g as i32 - other[1]).pow(2)
+            + (self.rgb.b as i32 - other[2]).pow(2)
+    }
+
+    fn contains_point(&self, other: &[i32; 3]) -> bool {
+        &[self.rgb.r as i32, self.rgb.g as i32, self.rgb.b as i32] == other
+    }
+}
+
+struct TestParams;
+impl RTreeParams for TestParams {
+    type DefaultInsertionStrategy = rstar::RStarInsertionStrategy;
+
+    const MAX_SIZE: usize = 4;
+    const MIN_SIZE: usize = 2;
+    const REINSERTION_COUNT: usize = 1;
+}
+
+lazy_static::lazy_static! {
+    static ref PALETTE_RTREE: HashMap<BlockKind, RTree<ColorPoint, TestParams>> = {
+        let ron_bytes = include_bytes!("palettes.ron");
+        let palettes: HashMap<BlockKind, Vec<Rgb<u8>>> =
+            ron::de::from_bytes(&*ron_bytes).expect("palette should parse");
+        palettes
+            .into_iter()
+            .map(|(k, v)| {
+                let tree = RTree::bulk_load_with_params(v.into_iter()
+                    .enumerate()
+                    .map(|(index, rgb)| ColorPoint { rgb, index: index as u8 })
+                    .collect()
+                );
+                (k, tree)
+            })
+            .collect()
+    };
+    pub static ref PALETTE_KDTREE: HashMap<BlockKind, KdTree<f32, u8, 3>> = {
+        let ron_bytes = include_bytes!("palettes.ron");
+        let palettes: HashMap<BlockKind, Vec<Rgb<u8>>> =
+            ron::de::from_bytes(&*ron_bytes).expect("palette should parse");
+        palettes
+            .into_iter()
+            .map(|(k, v)| {
+                let mut tree = KdTree::new();
+                for (i, rgb) in v.into_iter().enumerate() {
+                    tree.add(&[rgb.r as f32, rgb.g as f32, rgb.b as f32], i as u8)
+                        .expect("kdtree insert should succeed");
+                }
+                (k, tree)
+            })
+            .collect()
+    };
+}
+
+pub trait NearestNeighbor {
+    fn nearest_neighbor(&self, x: &Rgb<u8>) -> Option<u8>;
+}
+
+impl NearestNeighbor for KdTree<f32, u8, 3> {
+    fn nearest_neighbor(&self, x: &Rgb<u8>) -> Option<u8> {
+        self.nearest_one(
+            &[x.r as f32, x.g as f32, x.b as f32],
+            &kiddo::distance::squared_euclidean,
+        )
+        .map(|(_, i)| *i)
+        .ok()
+    }
+}
+
+impl<P: RTreeParams> NearestNeighbor for RTree<ColorPoint, P> {
+    fn nearest_neighbor(&self, x: &Rgb<u8>) -> Option<u8> {
+        self.nearest_neighbor(&[x.r as i32, x.g as i32, x.b as i32])
+            .map(|p| p.index)
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct PaletteEncoding<'a, NN: NearestNeighbor, const N: u32>(&'a HashMap<BlockKind, NN>);
+
+impl<'a, NN: NearestNeighbor, const N: u32> VoxelImageEncoding for PaletteEncoding<'a, NN, N> {
+    #[allow(clippy::type_complexity)]
+    type Output = CompressedData<(Vec<u8>, [usize; 4])>;
+    #[allow(clippy::type_complexity)]
+    type Workspace = (
+        ImageBuffer<image::Luma<u8>, Vec<u8>>,
+        ImageBuffer<image::Luma<u8>, Vec<u8>>,
+        ImageBuffer<image::Luma<u8>, Vec<u8>>,
+        ImageBuffer<image::Luma<u8>, Vec<u8>>,
+    );
+
+    fn create(width: u32, height: u32) -> Self::Workspace {
+        (
+            ImageBuffer::new(width, height),
+            ImageBuffer::new(width, height),
+            ImageBuffer::new(width, height),
+            ImageBuffer::new(width / N, height / N),
+        )
+    }
+
+    fn put_solid(&self, ws: &mut Self::Workspace, x: u32, y: u32, kind: BlockKind, rgb: Rgb<u8>) {
+        ws.0.put_pixel(x, y, image::Luma([kind as u8]));
+        let i = self.0[&kind].nearest_neighbor(&rgb).unwrap_or(0);
+        ws.3.put_pixel(x / N, y / N, image::Luma([i]));
+    }
+
+    fn put_sprite(
+        &self,
+        ws: &mut Self::Workspace,
+        x: u32,
+        y: u32,
+        kind: BlockKind,
+        sprite: SpriteKind,
+        ori: Option<u8>,
+    ) {
+        ws.0.put_pixel(x, y, image::Luma([kind as u8]));
+        ws.1.put_pixel(x, y, image::Luma([sprite as u8]));
+        ws.2.put_pixel(x, y, image::Luma([ori.unwrap_or(0)]));
+    }
+
+    fn finish(ws: &Self::Workspace) -> Option<Self::Output> {
+        let mut buf = Vec::new();
+        use image::codecs::png::{CompressionType, FilterType};
+        let mut indices = [0; 4];
+        let mut f = |x: &ImageBuffer<_, Vec<u8>>, i| {
+            let png = image::codecs::png::PngEncoder::new_with_quality(
+                &mut buf,
+                CompressionType::Rle,
+                FilterType::Up,
+            );
+            png.encode(&*x.as_raw(), x.width(), x.height(), image::ColorType::L8)
+                .ok()?;
+            indices[i] = buf.len();
+            Some(())
+        };
+        f(&ws.0, 0)?;
+        f(&ws.1, 1)?;
+        f(&ws.2, 2)?;
+        f(&ws.3, 3)?;
+
+        Some(CompressedData::compress(&(buf, indices), 1))
+    }
+}
+
 #[allow(clippy::many_single_char_names)]
 fn histogram_to_dictionary(histogram: &HashMap<Vec<u8>, usize>, dictionary: &mut Vec<u8>) {
     let mut tmp: Vec<(Vec<u8>, usize)> = histogram.iter().map(|(k, v)| (k.clone(), *v)).collect();
@@ -778,7 +945,7 @@ fn main() {
                 if !SKIP_IMAGECHONK {
                     let jpegchonkgrid_pre = Instant::now();
                     let jpegchonkgrid =
-                        image_terrain_chonk(JpegEncoding, GridLtrPacking, &chunk).unwrap();
+                        image_terrain_chonk(&JpegEncoding, GridLtrPacking, &chunk).unwrap();
                     let jpegchonkgrid_post = Instant::now();
 
                     if false {
@@ -793,19 +960,19 @@ fn main() {
 
                     let jpegchonktall_pre = Instant::now();
                     let jpegchonktall =
-                        image_terrain_chonk(JpegEncoding, TallPacking { flip_y: false }, &chunk)
+                        image_terrain_chonk(&JpegEncoding, TallPacking { flip_y: false }, &chunk)
                             .unwrap();
                     let jpegchonktall_post = Instant::now();
 
                     let jpegchonkflip_pre = Instant::now();
                     let jpegchonkflip =
-                        image_terrain_chonk(JpegEncoding, TallPacking { flip_y: true }, &chunk)
+                        image_terrain_chonk(&JpegEncoding, TallPacking { flip_y: true }, &chunk)
                             .unwrap();
                     let jpegchonkflip_post = Instant::now();
 
                     let pngchonk_pre = Instant::now();
                     let pngchonk =
-                        image_terrain_chonk(PngEncoding, GridLtrPacking, &chunk).unwrap();
+                        image_terrain_chonk(&PngEncoding, GridLtrPacking, &chunk).unwrap();
                     let pngchonk_post = Instant::now();
 
                     sizes.extend_from_slice(&[
@@ -825,7 +992,7 @@ fn main() {
                 if !SKIP_MIXED {
                     let mixedchonk_pre = Instant::now();
                     let mixedchonk =
-                        image_terrain_chonk(MixedEncoding, TallPacking { flip_y: true }, &chunk)
+                        image_terrain_chonk(&MixedEncoding, TallPacking { flip_y: true }, &chunk)
                             .unwrap();
                     let mixedchonk_post = Instant::now();
 
@@ -834,7 +1001,7 @@ fn main() {
 
                     let mixeddense_pre = Instant::now();
                     let mixeddense = image_terrain_chonk(
-                        MixedEncodingDenseSprites,
+                        &MixedEncodingDenseSprites,
                         TallPacking { flip_y: true },
                         &chunk,
                     )
@@ -855,7 +1022,7 @@ fn main() {
 
                 let quadpngfull_pre = Instant::now();
                 let quadpngfull = image_terrain_chonk(
-                    QuadPngEncoding::<1>(),
+                    &QuadPngEncoding::<1>(),
                     TallPacking { flip_y: true },
                     &chunk,
                 )
@@ -864,7 +1031,7 @@ fn main() {
 
                 let quadpnghalf_pre = Instant::now();
                 let quadpnghalf = image_terrain_chonk(
-                    QuadPngEncoding::<2>(),
+                    &QuadPngEncoding::<2>(),
                     TallPacking { flip_y: true },
                     &chunk,
                 )
@@ -873,7 +1040,7 @@ fn main() {
 
                 let quadpngquarttall_pre = Instant::now();
                 let quadpngquarttall = image_terrain_chonk(
-                    QuadPngEncoding::<4>(),
+                    &QuadPngEncoding::<4>(),
                     TallPacking { flip_y: true },
                     &chunk,
                 )
@@ -882,22 +1049,40 @@ fn main() {
 
                 let quadpngquartwide_pre = Instant::now();
                 let quadpngquartwide =
-                    image_terrain_chonk(QuadPngEncoding::<4>(), WidePacking::<true>(), &chunk)
+                    image_terrain_chonk(&QuadPngEncoding::<4>(), WidePacking::<true>(), &chunk)
                         .unwrap();
                 let quadpngquartwide_post = Instant::now();
 
                 let tripngaverage_pre = Instant::now();
                 let tripngaverage =
-                    image_terrain_chonk(TriPngEncoding::<true>(), WidePacking::<true>(), &chunk)
+                    image_terrain_chonk(&TriPngEncoding::<true>(), WidePacking::<true>(), &chunk)
                         .unwrap();
                 let tripngaverage_post = Instant::now();
 
                 let tripngconst_pre = Instant::now();
                 let tripngconst =
-                    image_terrain_chonk(TriPngEncoding::<false>(), WidePacking::<true>(), &chunk)
+                    image_terrain_chonk(&TriPngEncoding::<false>(), WidePacking::<true>(), &chunk)
                         .unwrap();
                 let tripngconst_post = Instant::now();
 
+                let palette_kdtree_pre = Instant::now();
+                let palette_kdtree = image_terrain_chonk(
+                    &PaletteEncoding::<_, 4>(&PALETTE_KDTREE),
+                    WidePacking::<true>(),
+                    &chunk,
+                )
+                .unwrap();
+                let palette_kdtree_post = Instant::now();
+
+                let palette_rtree_pre = Instant::now();
+                let palette_rtree = image_terrain_chonk(
+                    &PaletteEncoding::<_, 4>(&PALETTE_RTREE),
+                    WidePacking::<true>(),
+                    &chunk,
+                )
+                .unwrap();
+                let palette_rtree_post = Instant::now();
+
                 #[rustfmt::skip]
                 sizes.extend_from_slice(&[
                     ("quadpngfull", quadpngfull.data.len() as f32 / n as f32),
@@ -906,6 +1091,8 @@ fn main() {
                     ("quadpngquartwide", quadpngquartwide.data.len() as f32 / n as f32),
                     ("tripngaverage", tripngaverage.data.len() as f32 / n as f32),
                     ("tripngconst", tripngconst.data.len() as f32 / n as f32),
+                    ("palette_kdtree", palette_kdtree.data.len() as f32 / n as f32),
+                    ("palette_rtree", palette_rtree.data.len() as f32 / n as f32),
                 ]);
                 let best_idx = sizes
                     .iter()
@@ -926,6 +1113,8 @@ fn main() {
                     ("quadpngquartwide", (quadpngquartwide_post - quadpngquartwide_pre).subsec_nanos()),
                     ("tripngaverage", (tripngaverage_post - tripngaverage_pre).subsec_nanos()),
                     ("tripngconst", (tripngconst_post - tripngconst_pre).subsec_nanos()),
+                    ("palette_kdtree", (palette_kdtree_post - palette_kdtree_pre).subsec_nanos()),
+                    ("palette_rtree", (palette_rtree_post - palette_rtree_pre).subsec_nanos()),
                 ]);
                 if false {
                     let bucket = z_buckets
@@ -965,6 +1154,24 @@ fn main() {
                     bucket.0 += 1;
                     bucket.1 += (tripngconst_post - tripngconst_pre).subsec_nanos() as f32;
                 }
+                if true {
+                    let bucket = z_buckets
+                        .entry("palette_kdtree")
+                        .or_default()
+                        .entry(chunk.get_max_z() - chunk.get_min_z())
+                        .or_insert((0, 0.0));
+                    bucket.0 += 1;
+                    bucket.1 += (palette_kdtree_post - palette_kdtree_pre).subsec_nanos() as f32;
+                }
+                if true {
+                    let bucket = z_buckets
+                        .entry("palette_rtree")
+                        .or_default()
+                        .entry(chunk.get_max_z() - chunk.get_min_z())
+                        .or_insert((0, 0.0));
+                    bucket.0 += 1;
+                    bucket.1 += (palette_rtree_post - palette_rtree_pre).subsec_nanos() as f32;
+                }
                 trace!(
                     "{} {}: uncompressed: {}, {:?} {} {:?}",
                     spiralpos.x,
@@ -992,12 +1199,13 @@ fn main() {
                         let mut f = File::create(&format!("chonkjpegs/{}_{}.jpg", sitename, count))
                             .unwrap();
                         let jpeg_volgrid =
-                            image_terrain_volgrid(JpegEncoding, GridLtrPacking, &volgrid).unwrap();
+                            image_terrain_volgrid(&JpegEncoding, GridLtrPacking, &volgrid).unwrap();
                         f.write_all(&*jpeg_volgrid).unwrap();
 
                         let mixedgrid_pre = Instant::now();
                         let (mixed_volgrid, indices) =
-                            image_terrain_volgrid(MixedEncoding, GridLtrPacking, &volgrid).unwrap();
+                            image_terrain_volgrid(&MixedEncoding, GridLtrPacking, &volgrid)
+                                .unwrap();
                         let mixedgrid_post = Instant::now();
                         let seconds = (mixedgrid_post - mixedgrid_pre).as_secs_f64();
                         println!(

world/examples/pricing_csv.rs

@@ -73,7 +73,7 @@ fn economy_sqlite(world: &World, index: &Index) -> Result<(), Box<dyn Error>> {
     DROP TABLE IF EXISTS site;
     CREATE TABLE site (
         xcoord INTEGER NOT NULL,
-        ycoord INTEGER NUT NULL,
+        ycoord INTEGER NOT NULL,
         name TEXT NOT NULL
     );
     CREATE UNIQUE INDEX site_position ON site(xcoord, ycoord);

world/examples/world_block_statistics.rs

@@ -0,0 +1,284 @@
+use clap::{App, Arg, SubCommand};
+use common::{
+    terrain::{BlockKind, TerrainChunkSize},
+    vol::{IntoVolIterator, RectVolSize},
+};
+use fallible_iterator::FallibleIterator;
+use kiddo::{distance::squared_euclidean, KdTree};
+use rayon::{
+    iter::{IntoParallelIterator, ParallelIterator},
+    ThreadPoolBuilder,
+};
+use rusqlite::{Connection, ToSql, Transaction, TransactionBehavior, NO_PARAMS};
+use std::{
+    collections::{HashMap, HashSet},
+    error::Error,
+    fs::File,
+    io::Write,
+    str::FromStr,
+    sync::mpsc,
+    time::{SystemTime, UNIX_EPOCH},
+};
+use vek::*;
+use veloren_world::{
+    sim::{FileOpts, WorldOpts, DEFAULT_WORLD_MAP},
+    World,
+};
+
+fn block_statistics_db(db_path: &str) -> Result<Connection, Box<dyn Error>> {
+    let conn = Connection::open(db_path)?;
+    #[rustfmt::skip]
+    conn.execute_batch("
+    CREATE TABLE IF NOT EXISTS chunk (
+        xcoord INTEGER NOT NULL,
+        ycoord INTEGER NOT NULL,
+        height INTEGER NOT NULL,
+        start_time REAL NOT NULL,
+        end_time REAL NOT NULL
+    );
+    CREATE UNIQUE INDEX IF NOT EXISTS chunk_position ON chunk(xcoord, ycoord);
+    CREATE TABLE IF NOT EXISTS block (
+        xcoord INTEGER NOT NULL,
+        ycoord INTEGER NOT NULL,
+        kind TEXT NOT NULL,
+        r INTEGER NOT NULL,
+        g INTEGER NOT NULL,
+        b INTEGER NOT NULL,
+        quantity INTEGER NOT NULL
+    );
+    CREATE UNIQUE INDEX IF NOT EXISTS block_position ON block(xcoord, ycoord, kind, r, g, b);
+    CREATE TABLE IF NOT EXISTS sprite (
+        xcoord INTEGER NOT NULL,
+        ycoord INTEGER NOT NULL,
+        kind TEXT NOT NULL,
+        quantity INTEGER NOT NULL
+    );
+    CREATE UNIQUE INDEX IF NOT EXISTS sprite_position ON sprite(xcoord, ycoord, kind);
+    ")?;
+    Ok(conn)
+}
+
+fn generate(db_path: &str, ymin: Option<i32>, ymax: Option<i32>) -> Result<(), Box<dyn Error>> {
+    common_frontend::init_stdout(None);
+    println!("Loading world");
+    let pool = ThreadPoolBuilder::new().build().unwrap();
+    let (world, index) = World::generate(
+        59686,
+        WorldOpts {
+            seed_elements: true,
+            world_file: FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into()),
+        },
+        &pool,
+    );
+    println!("Loaded world");
+
+    let conn = block_statistics_db(db_path)?;
+
+    let existing_chunks: HashSet<(i32, i32)> = conn
+        .prepare("SELECT xcoord, ycoord FROM chunk")?
+        .query(NO_PARAMS)?
+        .map(|row| Ok((row.get(0)?, row.get(1)?)))
+        .collect()?;
+
+    let sz = world.sim().get_size();
+    let (tx, rx) = mpsc::channel();
+    rayon::spawn(move || {
+        let coords: Vec<_> = (ymin.unwrap_or(1)..ymax.unwrap_or(sz.y as i32))
+            .into_iter()
+            .flat_map(move |y| {
+                let tx = tx.clone();
+                (1..sz.x as i32)
+                    .into_iter()
+                    .map(move |x| (tx.clone(), x, y))
+            })
+            .collect();
+        coords.into_par_iter().for_each(|(tx, x, y)| {
+            if existing_chunks.contains(&(x, y)) {
+                return;
+            }
+            println!("Generating chunk at ({}, {})", x, y);
+            let start_time = SystemTime::now();
+            if let Ok((chunk, _supplement)) =
+                world.generate_chunk(index.as_index_ref(), Vec2::new(x, y), || false, None)
+            {
+                let end_time = SystemTime::now();
+                // TODO: can kiddo be made to work without the `Float` bound, so we can use
+                // `KdTree<u8, (), 3>` (currently it uses 15 bytes per point instead of 3)?
+                let mut block_colors = KdTree::<f32, Rgb<u8>, 3>::new();
+                let mut block_counts = HashMap::new();
+                let mut sprite_counts = HashMap::new();
+                let lo = Vec3::new(0, 0, chunk.get_min_z());
+                let hi = TerrainChunkSize::RECT_SIZE.as_().with_z(chunk.get_max_z());
+                let height = chunk.get_max_z() - chunk.get_min_z();
+                for (_, block) in chunk.vol_iter(lo, hi) {
+                    let mut rgb = block.get_color().unwrap_or_else(|| Rgb::new(0, 0, 0));
+                    let color: [f32; 3] = [rgb.r as _, rgb.g as _, rgb.b as _];
+                    if let Ok((dist, nearest)) =
+                        block_colors.nearest_one(&color, &squared_euclidean)
+                    {
+                        if dist < (5.0f32).powf(2.0) {
+                            rgb = *nearest;
+                        }
+                    }
+                    let _ = block_colors.add(&color, rgb);
+                    *block_counts.entry((block.kind(), rgb)).or_insert(0) += 1;
+                    if let Some(sprite) = block.get_sprite() {
+                        *sprite_counts.entry(sprite).or_insert(0) += 1;
+                    }
+                }
+                let _ = tx.send((
+                    x,
+                    y,
+                    height,
+                    start_time,
+                    end_time,
+                    block_counts,
+                    sprite_counts,
+                ));
+            }
+        });
+    });
+    let mut tx = Transaction::new_unchecked(&conn, TransactionBehavior::Deferred)?;
+    let mut i = 0;
+    while let Ok((x, y, height, start_time, end_time, block_counts, sprite_counts)) = rx.recv() {
+        #[rustfmt::skip]
+        let mut insert_block = tx.prepare_cached("
+            REPLACE INTO block (xcoord, ycoord, kind, r, g, b, quantity)
+            VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+        ")?;
+        #[rustfmt::skip]
+        let mut insert_sprite = tx.prepare_cached("
+            REPLACE INTO sprite (xcoord, ycoord, kind, quantity)
+            VALUES (?1, ?2, ?3, ?4)
+        ")?;
+        #[rustfmt::skip]
+        let mut insert_chunk = tx.prepare_cached("
+            REPLACE INTO chunk (xcoord, ycoord, height, start_time, end_time)
+            VALUES (?1, ?2, ?3, ?4, ?5)
+        ")?;
+        println!("Inserting results for chunk at ({}, {}): {}", x, y, i);
+        for ((kind, color), count) in block_counts.iter() {
+            insert_block.execute(&[
+                &x as &dyn ToSql,
+                &y,
+                &format!("{:?}", kind),
+                &color.r,
+                &color.g,
+                &color.b,
+                &count,
+            ])?;
+        }
+        for (kind, count) in sprite_counts.iter() {
+            insert_sprite.execute(&[&x as &dyn ToSql, &y, &format!("{:?}", kind), &count])?;
+        }
+        let start_time = start_time.duration_since(UNIX_EPOCH)?.as_secs_f64();
+        let end_time = end_time.duration_since(UNIX_EPOCH)?.as_secs_f64();
+        insert_chunk.execute(&[&x as &dyn ToSql, &y, &height, &start_time, &end_time])?;
+        if i % 32 == 0 {
+            println!("Committing last 32 chunks");
+            drop(insert_block);
+            drop(insert_sprite);
+            drop(insert_chunk);
+            tx.commit()?;
+            tx = Transaction::new_unchecked(&conn, TransactionBehavior::Deferred)?;
+        }
+        i += 1;
+    }
+    Ok(())
+}
+
+fn palette(conn: Connection) -> Result<(), Box<dyn Error>> {
+    let mut stmt =
+        conn.prepare("SELECT kind, r, g, b, SUM(quantity) FROM block GROUP BY kind, r, g, b")?;
+    let mut block_colors: HashMap<BlockKind, Vec<(Rgb<u8>, i64)>> = HashMap::new();
+
+    let mut rows = stmt.query(NO_PARAMS)?;
+    while let Some(row) = rows.next()? {
+        let kind = BlockKind::from_str(&row.get::<_, String>(0)?)?;
+        let rgb: Rgb<u8> = Rgb::new(row.get(1)?, row.get(2)?, row.get(3)?);
+        let count: i64 = row.get(4)?;
+        block_colors
+            .entry(kind)
+            .or_insert_with(Vec::new)
+            .push((rgb, count));
+    }
+    for (_, v) in block_colors.iter_mut() {
+        v.sort_by(|a, b| b.1.cmp(&a.1));
+    }
+
+    let mut palettes: HashMap<BlockKind, Vec<Rgb<u8>>> = HashMap::new();
+    for (kind, colors) in block_colors.iter() {
+        let palette = palettes.entry(*kind).or_insert_with(Vec::new);
+        if colors.len() <= 256 {
+            for (color, _) in colors {
+                palette.push(*color);
+            }
+            println!("{:?}: {:?}", kind, palette);
+            continue;
+        }
+        let mut radius = 1024.0;
+        let mut tree = KdTree::<f32, Rgb<u8>, 3>::new();
+        while palette.len() < 256 {
+            if let Some((color, _)) = colors.iter().find(|(color, _)| {
+                tree.nearest_one(
+                    &[color.r as f32, color.g as f32, color.b as f32],
+                    &squared_euclidean,
+                )
+                .map(|(dist, _)| dist > radius)
+                .unwrap_or(true)
+            }) {
+                palette.push(*color);
+                tree.add(&[color.r as f32, color.g as f32, color.b as f32], *color)?;
+                println!("{:?}, {:?}: {:?}", kind, radius, *color);
+            } else {
+                radius -= 1.0;
+            }
+        }
+    }
+    let mut f = File::create("palettes.ron")?;
+    let pretty = ron::ser::PrettyConfig::default().with_depth_limit(2);
+    write!(f, "{}", ron::ser::to_string_pretty(&palettes, pretty)?)?;
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let mut app = App::new("world_block_statistics")
+        .version(common::util::DISPLAY_VERSION_LONG.as_str())
+        .author("The veloren devs <https://gitlab.com/veloren/veloren>")
+        .about("Compute and process block statistics on generated chunks")
+        .subcommand(
+            SubCommand::with_name("generate")
+                .about("Generate block statistics")
+                .args(&[
+                    Arg::with_name("database")
+                        .required(true)
+                        .help("File to generate/resume generation"),
+                    Arg::with_name("ymin").long("ymin").takes_value(true),
+                    Arg::with_name("ymax").long("ymax").takes_value(true),
+                ]),
+        )
+        .subcommand(
+            SubCommand::with_name("palette")
+                .about("Compute a palette from previously gathered statistics")
+                .args(&[Arg::with_name("database").required(true)]),
+        );
+
+    let matches = app.clone().get_matches();
+    match matches.subcommand() {
+        ("generate", Some(matches)) => {
+            let db_path = matches.value_of("database").expect("database is required");
+            let ymin = matches.value_of("ymin").and_then(|x| i32::from_str(x).ok());
+            let ymax = matches.value_of("ymax").and_then(|x| i32::from_str(x).ok());
+            generate(db_path, ymin, ymax)?;
+        },
+        ("palette", Some(matches)) => {
+            let conn =
+                Connection::open(&matches.value_of("database").expect("database is required"))?;
+            palette(conn)?;
+        },
+        _ => {
+            app.print_help()?;
+        },
+    }
+    Ok(())
+}