From 92fa0a4d08bdf960b2ee4b3a6e69caaf4fd33b6f Mon Sep 17 00:00:00 2001 From: Joshua Yanovski Date: Mon, 28 Sep 2020 16:43:23 +0200 Subject: [PATCH] Fix hacky solution with proper defragmentation. After generating a chonk, we now find the highest frequency block (in terms of the number of groups that uniformly consist of that block) and replace the chunk's default with that one. We also resort the data in the process to be in the same order as the original array index. This improves our memory savings from 3x to almost 7x, and brings us within a factor of 3 or so of what I hope a true average will be. The defragmentation is not totally optimal and can probably be improved from a performance perspective, but given how much of a hard bottleneck RAM is this seems worthwhile. Also, this doesn't suffer from the issues the previous solution did. --- common/src/terrain/block.rs | 2 +- common/src/terrain/chonk.rs | 15 +++--- common/src/volumes/chunk.rs | 95 ++++++++++++++++++++++++++++++++++--- world/src/lib.rs | 4 ++ 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/common/src/terrain/block.rs b/common/src/terrain/block.rs index d807d0f548..2758d3f42c 100644 --- a/common/src/terrain/block.rs +++ b/common/src/terrain/block.rs @@ -89,7 +89,7 @@ impl<'a> TryFrom<&'a str> for BlockKind { fn try_from(s: &'a str) -> Result { BLOCK_KINDS.get(s).copied().ok_or(()) } } -#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] pub struct Block { kind: BlockKind, attr: [u8; 3], diff --git a/common/src/terrain/chonk.rs b/common/src/terrain/chonk.rs index 9414055ff3..5fe51fdb1d 100644 --- a/common/src/terrain/chonk.rs +++ b/common/src/terrain/chonk.rs @@ -5,8 +5,8 @@ use crate::{ }, volumes::chunk::{Chunk, ChunkError, ChunkPosIter, ChunkVolIter}, }; +use core::{hash::Hash, marker::PhantomData}; use serde::{Deserialize, Serialize}; -use std::marker::PhantomData; use vek::*; #[derive(Debug)] @@ -84,6 +84,14 @@ impl Chonk { // Returns the z offset of the sub_chunk that contains layer z fn sub_chunk_min_z(&self, z: i32) -> i32 { z - self.sub_chunk_z(z) } + + /// Compress chunk by using more intelligent defaults. + pub fn defragment(&mut self) + where + V: Clone + Eq + Hash, + { + self.sub_chunks.iter_mut().for_each(SubChunk::defragment); + } } impl BaseVol for Chonk { @@ -130,11 +138,6 @@ impl WriteVol for Chonk self.z_offset += sub_chunk_idx * SubChunkSize::::SIZE.z as i32; sub_chunk_idx = 0; } else if pos.z >= self.get_max_z() { - if self.sub_chunks.is_empty() && block == self.below { - // Try not to generate extra blocks unless necessary. - self.z_offset += 1; - return Ok(()); - } // Append exactly sufficiently many SubChunks via Vec::extend let c = Chunk::, M>::filled(self.above.clone(), self.meta.clone()); let n = 1 + sub_chunk_idx as usize - self.sub_chunks.len(); diff --git a/common/src/volumes/chunk.rs b/common/src/volumes/chunk.rs index 86ecc5bf37..832fd44c87 100644 --- a/common/src/volumes/chunk.rs +++ b/common/src/volumes/chunk.rs @@ -1,8 +1,9 @@ use crate::vol::{ BaseVol, IntoPosIterator, IntoVolIterator, RasterableVol, ReadVol, VolSize, WriteVol, }; +use core::{hash::Hash, iter::Iterator, marker::PhantomData, mem}; +use hashbrown::HashMap; use serde::{Deserialize, Serialize}; -use std::{iter::Iterator, marker::PhantomData}; use vek::*; #[derive(Debug)] @@ -56,7 +57,7 @@ pub struct Chunk { } impl Chunk { - const GROUP_COUNT: Vec3 = Vec3::new( + pub const GROUP_COUNT: Vec3 = Vec3::new( S::SIZE.x / Self::GROUP_SIZE.x, S::SIZE.y / Self::GROUP_SIZE.y, S::SIZE.z / Self::GROUP_SIZE.z, @@ -115,6 +116,86 @@ impl Chunk { } } + /// Compress this subchunk by frequency. + pub fn defragment(&mut self) + where + V: Clone + Eq + Hash, + { + // First, construct a HashMap with max capacity equal to GROUP_COUNT (since each + // filled group can have at most one slot). + let mut map = HashMap::with_capacity(Self::GROUP_COUNT_TOTAL as usize); + let vox = &self.vox; + let default = &self.default; + self.indices + .iter() + .enumerate() + .for_each(|(grp_idx, &base)| { + let start = usize::from(base) * Self::GROUP_VOLUME as usize; + let end = start + Self::GROUP_VOLUME as usize; + if let Some(group) = vox.get(start..end) { + // Check to see if all blocks in this group are the same. + let mut group = group.iter(); + let first = group.next().expect("GROUP_VOLUME ≥ 1"); + if group.all(|block| block == first) { + // All blocks in the group were the same, so add our position to this entry + // in the HashMap. + map.entry(first).or_insert(vec![]).push(grp_idx); + } + } else { + // This slot is empty (i.e. has the default value). + map.entry(default).or_insert(vec![]).push(grp_idx); + } + }); + // Now, find the block with max frequency in the HashMap and make that our new + // default. + let (new_default, default_groups) = if let Some((new_default, default_groups)) = map + .into_iter() + .max_by_key(|(_, default_groups)| default_groups.len()) + { + (new_default.clone(), default_groups) + } else { + // There is no good choice for default group, so leave it as is. + return; + }; + + // For simplicity, we construct a completely new voxel array rather than + // attempting in-place updates (TODO: consider changing this). + let mut new_vox = + Vec::with_capacity(Self::GROUP_COUNT_TOTAL as usize - default_groups.len()); + let num_groups = self.num_groups(); + self.indices + .iter_mut() + .enumerate() + .for_each(|(grp_idx, base)| { + if default_groups.contains(&grp_idx) { + // Default groups become 255 + *base = 255; + } else { + // Other groups are allocated in increasing order by group index. + // NOTE: Cannot overflow since the current implicit group index can't be at the + // end of the vector until at the earliest after the 256th iteration. + let old_base = usize::from(mem::replace( + base, + (new_vox.len() / Self::GROUP_VOLUME as usize) as u8, + )); + if old_base >= num_groups { + // Old default, which (since we reached this branch) is not equal to the new + // default, so we have to write out the old default. + new_vox + .resize(new_vox.len() + Self::GROUP_VOLUME as usize, default.clone()); + } else { + let start = old_base * Self::GROUP_VOLUME as usize; + let end = start + Self::GROUP_VOLUME as usize; + new_vox.extend_from_slice(&vox[start..end]); + } + } + }); + + // Finally, reset our vox and default values to the new ones. + self.vox = new_vox; + self.default = new_default; + } + /// Get a reference to the internal metadata. pub fn metadata(&self) -> &M { &self.meta } @@ -143,12 +224,12 @@ impl Chunk { fn idx_unchecked(&self, pos: Vec3) -> Option { let grp_idx = Self::grp_idx(pos); let rel_idx = Self::rel_idx(pos); - let base = self.indices[grp_idx as usize]; + let base = u32::from(self.indices[grp_idx as usize]); let num_groups = self.vox.len() as u32 / Self::GROUP_VOLUME; - if base as u32 >= num_groups { + if base >= num_groups { None } else { - Some((base as u32 * Self::GROUP_VOLUME + rel_idx) as usize) + Some((base * Self::GROUP_VOLUME + rel_idx) as usize) } } @@ -161,12 +242,12 @@ impl Chunk { let rel_idx = Self::rel_idx(pos); let base = &mut self.indices[grp_idx as usize]; let num_groups = self.vox.len() as u32 / Self::GROUP_VOLUME; - if *base as u32 >= num_groups { + if u32::from(*base) >= num_groups { *base = num_groups as u8; self.vox .extend(std::iter::repeat(self.default.clone()).take(Self::GROUP_VOLUME as usize)); } - (*base as u32 * Self::GROUP_VOLUME + rel_idx) as usize + (u32::from(*base) * Self::GROUP_VOLUME + rel_idx) as usize } #[inline(always)] diff --git a/world/src/lib.rs b/world/src/lib.rs index bc2b4d3477..d5fc53736f 100644 --- a/world/src/lib.rs +++ b/world/src/lib.rs @@ -153,6 +153,7 @@ impl World { let meta = TerrainChunkMeta::new(sim_chunk.get_name(&self.sim), sim_chunk.get_biome()); let mut chunk = TerrainChunk::new(base_z, stone, air, meta); + for y in 0..TerrainChunkSize::RECT_SIZE.y as i32 { for x in 0..TerrainChunkSize::RECT_SIZE.x as i32 { if should_continue() { @@ -316,6 +317,9 @@ impl World { ) }); + // Finally, defragment to minimize space consumption. + chunk.defragment(); + Ok((chunk, supplement)) } }