From 6d9de520f31a386ede4f27f4691e37825c853fac Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Tue, 20 Apr 2021 17:33:46 -0400 Subject: [PATCH 01/41] Compress terrain chunks with deflate. Includes a benchmark showing that this makes them around 70% smaller, and is the same speed as LZ4. --- CHANGELOG.md | 1 + Cargo.lock | 17 +- client/src/lib.rs | 2 +- common/Cargo.toml | 5 +- common/net/src/msg/server.rs | 4 +- common/src/terrain/mod.rs | 51 ++++ server/src/sys/msg/terrain.rs | 4 +- server/src/sys/terrain.rs | 24 +- server/src/sys/terrain_sync.rs | 9 +- world/Cargo.toml | 11 + world/src/bin/chunk_compression_benchmarks.rs | 246 ++++++++++++++++++ 11 files changed, 363 insertions(+), 11 deletions(-) create mode 100644 world/src/bin/chunk_compression_benchmarks.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cb78c2f892..65067d5942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Entities now have density - Buoyancy is calculated from the difference in density between an entity and surrounding fluid - Drag is now calculated based on physical properties +- Terrain chunks are now deflate-compressed when sent over the network. ### Changed diff --git a/Cargo.lock b/Cargo.lock index ef8a66c119..e41c084765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1308,6 +1308,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "deflate" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f95bf05dffba6e6cce8dfbb30def788154949ccd9aed761b472119c21e01c70" +dependencies = [ + "adler32", +] + [[package]] name = "derivative" version = "2.2.0" @@ -3765,7 +3774,7 @@ checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" dependencies = [ "bitflags", "crc32fast", - "deflate", + "deflate 0.8.6", "miniz_oxide 0.3.7", ] @@ -5454,6 +5463,7 @@ dependencies = [ "approx 0.4.0", "arraygen", "assets_manager", + "bincode", "bitflags", "criterion", "crossbeam-channel", @@ -5461,6 +5471,7 @@ dependencies = [ "csv", "dot_vox", "enum-iterator", + "flate2", "hashbrown", "image", "indexmap", @@ -5809,12 +5820,15 @@ dependencies = [ "bincode", "bitvec", "criterion", + "deflate 0.9.1", "enum-iterator", + "flate2", "fxhash", "hashbrown", "image", "itertools 0.10.0", "lazy_static", + "lz-fear", "minifb", "noise", "num 0.4.0", @@ -5831,6 +5845,7 @@ dependencies = [ "tracing-subscriber", "vek", "veloren-common", + "veloren-common-frontend", "veloren-common-net", ] diff --git a/client/src/lib.rs b/client/src/lib.rs index 39ce8d113e..7a6ecf3d41 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1923,7 +1923,7 @@ impl Client { fn handle_server_terrain_msg(&mut self, msg: ServerGeneral) -> Result<(), Error> { match msg { ServerGeneral::TerrainChunkUpdate { key, chunk } => { - if let Ok(chunk) = chunk { + if let Some(chunk) = chunk.ok().and_then(|c| c.to_chunk()) { self.state.insert_chunk(key, chunk); } self.pending_chunks.remove(&key); diff --git a/common/Cargo.toml b/common/Cargo.toml index 8c3ef5d6c9..5028493a73 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -9,8 +9,9 @@ no-assets = [] tracy = ["common-base/tracy"] simd = ["vek/platform_intrinsics"] bin_csv = ["csv", "structopt"] +compression = ["flate2"] -default = ["simd"] +default = ["simd", "compression"] [dependencies] @@ -23,11 +24,13 @@ serde = { version = "1.0.110", features = ["derive", "rc"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] approx = "0.4.0" arraygen = "0.1.13" +bincode = "1.3.3" crossbeam-utils = "0.8.1" bitflags = "1.2" crossbeam-channel = "0.5" enum-iterator = "0.6" lazy_static = "1.4.0" +flate2 = { version = "1.0.20", optional = true } num-derive = "0.3" num-traits = "0.2" ordered-float = { version = "2.0.1", default-features = false } diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs index 239d3e10d0..1a656e08c5 100644 --- a/common/net/src/msg/server.rs +++ b/common/net/src/msg/server.rs @@ -6,7 +6,7 @@ use common::{ outcome::Outcome, recipe::RecipeBook, resources::TimeOfDay, - terrain::{Block, TerrainChunk}, + terrain::{Block, SerializedTerrainChunk}, trade::{PendingTrade, SitePrices, TradeId, TradeResult}, uid::Uid, }; @@ -106,7 +106,7 @@ pub enum ServerGeneral { // Ingame related AND terrain stream TerrainChunkUpdate { key: Vec2, - chunk: Result, ()>, + chunk: Result, }, TerrainBlockUpdates(HashMap, Block>), // Always possible diff --git a/common/src/terrain/mod.rs b/common/src/terrain/mod.rs index 5360b6c7c8..f1fb3a4106 100644 --- a/common/src/terrain/mod.rs +++ b/common/src/terrain/mod.rs @@ -17,6 +17,7 @@ pub use self::{ }; use roots::find_roots_cubic; use serde::{Deserialize, Serialize}; +use tracing::trace; use crate::{ vol::{ReadVol, RectVolSize}, @@ -142,6 +143,56 @@ impl TerrainChunkMeta { pub type TerrainChunk = chonk::Chonk; pub type TerrainGrid = VolGrid2d; +/// Wrapper for custom serialization strategies (e.g. compression) for terrain +/// chunks +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SerializedTerrainChunk(pub Vec); + +impl SerializedTerrainChunk { + pub fn from_chunk(chunk: &TerrainChunk) -> Self { + let uncompressed = bincode::serialize(chunk) + .expect("bincode serialization can only fail if a byte limit is set"); + #[cfg(feature = "compression")] + { + use flate2::{write::DeflateEncoder, Compression}; + use std::io::Write; + const EXPECT_MSG: &str = + "compression only fails for fallible Read/Write impls (which Vec is not)"; + + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(5)); + encoder.write_all(&*uncompressed).expect(EXPECT_MSG); + let compressed = encoder.finish().expect(EXPECT_MSG); + trace!( + "compressed {}, uncompressed {}, ratio {}", + compressed.len(), + uncompressed.len(), + compressed.len() as f32 / uncompressed.len() as f32 + ); + SerializedTerrainChunk(compressed) + } + #[cfg(not(feature = "compression"))] + { + SerializedTerrainChunk(uncompressed) + } + } + + pub fn to_chunk(&self) -> Option { + #[cfg(feature = "compression")] + { + use std::io::Read; + let mut uncompressed = Vec::new(); + flate2::read::DeflateDecoder::new(&*self.0) + .read_to_end(&mut uncompressed) + .ok()?; + bincode::deserialize(&*uncompressed).ok() + } + #[cfg(not(feature = "compression"))] + { + bincode::deserialize(&self.0).ok() + } + } +} + impl TerrainGrid { /// Find a location suitable for spawning an entity near the given /// position (but in the same chunk). diff --git a/server/src/sys/msg/terrain.rs b/server/src/sys/msg/terrain.rs index 1dd5f82ecb..9725910373 100644 --- a/server/src/sys/msg/terrain.rs +++ b/server/src/sys/msg/terrain.rs @@ -2,7 +2,7 @@ use crate::{client::Client, metrics::NetworkRequestMetrics, presence::Presence}; use common::{ comp::Pos, event::{EventBus, ServerEvent}, - terrain::{TerrainChunkSize, TerrainGrid}, + terrain::{SerializedTerrainChunk, TerrainChunkSize, TerrainGrid}, vol::RectVolSize, }; use common_ecs::{Job, Origin, ParMode, Phase, System}; @@ -80,7 +80,7 @@ impl<'a> System<'a> for Sys { network_metrics.chunks_served_from_memory.inc(); client.send(ServerGeneral::TerrainChunkUpdate { key, - chunk: Ok(Arc::clone(chunk)), + chunk: Ok(SerializedTerrainChunk::from_chunk(&chunk)), })? }, None => { diff --git a/server/src/sys/terrain.rs b/server/src/sys/terrain.rs index 1d3f224828..3f7cbb145f 100644 --- a/server/src/sys/terrain.rs +++ b/server/src/sys/terrain.rs @@ -10,7 +10,7 @@ use common::{ event::{EventBus, ServerEvent}, generation::get_npc_name, npc::NPC_NAMES, - terrain::TerrainGrid, + terrain::{SerializedTerrainChunk, TerrainGrid}, LoadoutBuilder, SkillSetBuilder, }; use common_ecs::{Job, Origin, Phase, System}; @@ -93,6 +93,28 @@ impl<'a> System<'a> for Sys { // Add to list of chunks to send to nearby players. new_chunks.push((key, Arc::clone(&chunk))); + // Send the chunk to all nearby players. + let mut lazy_msg = None; + for (presence, pos, client) in (&presences, &positions, &clients).join() { + let chunk_pos = terrain.pos_key(pos.0.map(|e| e as i32)); + // Subtract 2 from the offset before computing squared magnitude + // 1 since chunks need neighbors to be meshed + // 1 to act as a buffer if the player moves in that direction + let adjusted_dist_sqr = (chunk_pos - key) + .map(|e: i32| (e.abs() as u32).saturating_sub(2)) + .magnitude_squared(); + + if adjusted_dist_sqr <= presence.view_distance.pow(2) { + if lazy_msg.is_none() { + lazy_msg = Some(client.prepare(ServerGeneral::TerrainChunkUpdate { + key, + chunk: Ok(SerializedTerrainChunk::from_chunk(&*chunk)), + })); + } + lazy_msg.as_ref().map(|ref msg| client.send_prepared(&msg)); + } + } + // TODO: code duplication for chunk insertion between here and state.rs // Insert the chunk into terrain changes if terrain.insert(key, chunk).is_some() { diff --git a/server/src/sys/terrain_sync.rs b/server/src/sys/terrain_sync.rs index 77f7cd1a86..9f203874f9 100644 --- a/server/src/sys/terrain_sync.rs +++ b/server/src/sys/terrain_sync.rs @@ -1,5 +1,8 @@ use crate::{client::Client, presence::Presence}; -use common::{comp::Pos, terrain::TerrainGrid}; +use common::{ + comp::Pos, + terrain::{SerializedTerrainChunk, TerrainGrid}, +}; use common_ecs::{Job, Origin, Phase, System}; use common_net::msg::ServerGeneral; use common_state::TerrainChanges; @@ -38,8 +41,8 @@ impl<'a> System<'a> for Sys { if lazy_msg.is_none() { lazy_msg = Some(client.prepare(ServerGeneral::TerrainChunkUpdate { key: *chunk_key, - chunk: Ok(match terrain.get_key_arc(*chunk_key) { - Some(chunk) => Arc::clone(chunk), + chunk: Ok(match terrain.get_key(*chunk_key) { + Some(chunk) => SerializedTerrainChunk::from_chunk(&chunk), None => break 'chunk, }), })); diff --git a/world/Cargo.toml b/world/Cargo.toml index bc82bd6e45..e08f498f08 100644 --- a/world/Cargo.toml +++ b/world/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [features] tracy = ["common/tracy", "common-net/tracy"] simd = ["vek/platform_intrinsics"] +bin_compression = ["lz-fear", "deflate", "flate2", "common-frontend"] default = ["simd"] @@ -37,6 +38,12 @@ ron = { version = "0.6", default-features = false } assets_manager = {version = "0.4.3", features = ["ron"]} #inline_tweak = "1.0.2" +# compression benchmarks +lz-fear = { version = "0.1.1", optional = true } +deflate = { version = "0.9.1", optional = true } +flate2 = { version = "1.0.20", optional = true } +common-frontend = { package = "veloren-common-frontend", path = "../common/frontend", optional = true } + [dev-dependencies] criterion = "0.3" @@ -48,3 +55,7 @@ structopt = "0.3" [[bench]] harness = false name = "tree" + +[[bin]] +name = "chunk_compression_benchmarks" +required-features = ["bin_compression"] diff --git a/world/src/bin/chunk_compression_benchmarks.rs b/world/src/bin/chunk_compression_benchmarks.rs new file mode 100644 index 0000000000..a20cb03e53 --- /dev/null +++ b/world/src/bin/chunk_compression_benchmarks.rs @@ -0,0 +1,246 @@ +use common::{ + terrain::{chonk::Chonk, Block, BlockKind, SpriteKind}, + vol::{IntoVolIterator, RectVolSize, SizedVol, WriteVol}, + volumes::dyna::{Access, ColumnAccess, Dyna}, +}; +use hashbrown::HashMap; +use std::{ + io::{Read, Write}, + time::Instant, +}; +use tracing::{debug, trace}; +use vek::*; +use veloren_world::{ + sim::{FileOpts, WorldOpts, DEFAULT_WORLD_MAP}, + World, +}; + +fn lz4_with_dictionary(data: &[u8], dictionary: &[u8]) -> Vec { + let mut compressed = Vec::new(); + lz_fear::CompressionSettings::default() + .dictionary(0, &dictionary) + .compress(data, &mut compressed) + .unwrap(); + compressed +} + +#[allow(dead_code)] +fn unlz4_with_dictionary(data: &[u8], dictionary: &[u8]) -> Option> { + lz_fear::LZ4FrameReader::new(data).ok().and_then(|r| { + let mut uncompressed = Vec::new(); + r.into_read_with_dictionary(dictionary) + .read_to_end(&mut uncompressed) + .ok()?; + bincode::deserialize(&*uncompressed).ok() + }) +} + +#[allow(dead_code)] +fn do_deflate(data: &[u8]) -> Vec { + use deflate::{write::DeflateEncoder, Compression}; + + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::Fast); + encoder.write_all(data).expect("Write error!"); + let compressed_data = encoder.finish().expect("Failed to finish compression!"); + compressed_data +} + +fn do_deflate_flate2(data: &[u8]) -> Vec { + use flate2::{write::DeflateEncoder, Compression}; + + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(5)); + encoder.write_all(data).expect("Write error!"); + let compressed_data = encoder.finish().expect("Failed to finish compression!"); + compressed_data +} + +fn chonk_to_dyna( + chonk: &Chonk, + block: V, +) -> Dyna { + let mut dyna = Dyna::::filled( + Vec3::new( + S::RECT_SIZE.x, + S::RECT_SIZE.y, + (chonk.get_max_z() - chonk.get_min_z()) as u32, + ), + block, + chonk.meta().clone(), + ); + for (pos, block) in chonk.vol_iter( + Vec3::new(0, 0, chonk.get_min_z()), + Vec3::new(S::RECT_SIZE.x as _, S::RECT_SIZE.y as _, chonk.get_max_z()), + ) { + dyna.set(pos - chonk.get_min_z() * Vec3::unit_z(), block.clone()) + .expect("a bug here represents the arithmetic being wrong"); + } + dyna +} + +fn channelize_dyna( + dyna: &Dyna, +) -> ( + Dyna, + Vec, + Vec, + Vec, + Vec, +) { + let mut blocks = Dyna::filled(dyna.sz, BlockKind::Air, dyna.metadata().clone()); + let (mut r, mut g, mut b, mut sprites) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for (pos, block) in dyna.vol_iter(dyna.lower_bound(), dyna.upper_bound()) { + blocks.set(pos, **block).unwrap(); + match (block.get_color(), block.get_sprite()) { + (Some(rgb), None) => { + r.push(rgb.r); + g.push(rgb.g); + b.push(rgb.b); + }, + (None, Some(spritekind)) => { + sprites.push(spritekind); + }, + _ => panic!( + "attr being used for color vs sprite is mutually exclusive (and that's required \ + for this translation to be lossless), but there's no way to guarantee that at \ + the type level with Block's public API" + ), + } + } + (blocks, r, g, b, sprites) +} + +fn histogram_to_dictionary(histogram: &HashMap, usize>, dictionary: &mut Vec) { + let mut tmp: Vec<(Vec, usize)> = histogram.iter().map(|(k, v)| (k.clone(), *v)).collect(); + tmp.sort_by_key(|(_, count)| *count); + debug!("{:?}", tmp.last()); + let mut i = 0; + let mut j = tmp.len() - 1; + while i < dictionary.len() && j > 0 { + let (k, v) = &tmp[j]; + let dlen = dictionary.len(); + let n = (i + k.len()).min(dlen); + dictionary[i..n].copy_from_slice(&k[0..k.len().min(dlen - i)]); + debug!("{}: {}: {:?}", tmp.len() - j, v, k); + j -= 1; + i = n; + } +} + +fn main() { + common_frontend::init_stdout(None); + println!("Loading world"); + let (world, index) = World::generate(59686, WorldOpts { + seed_elements: true, + world_file: FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into()), + ..WorldOpts::default() + }); + println!("Loaded world"); + let mut histogram: HashMap, usize> = HashMap::new(); + let mut histogram2: HashMap, usize> = HashMap::new(); + let mut dictionary = vec![0xffu8; 1 << 16]; + let mut dictionary2 = vec![0xffu8; 1 << 16]; + let k = 32; + let sz = world.sim().get_size(); + let mut totals = [0.0; 5]; + let mut total_timings = [0.0; 2]; + let mut count = 0; + for y in 1..sz.y { + for x in 1..sz.x { + let chunk = + world.generate_chunk(index.as_index_ref(), Vec2::new(x as _, y as _), || false); + if let Ok((chunk, _)) = chunk { + let uncompressed = bincode::serialize(&chunk).unwrap(); + for w in uncompressed.windows(k) { + *histogram.entry(w.to_vec()).or_default() += 1; + } + if x % 128 == 0 { + histogram_to_dictionary(&histogram, &mut dictionary); + } + let lz4chonk_pre = Instant::now(); + let lz4_chonk = lz4_with_dictionary(&bincode::serialize(&chunk).unwrap(), &[]); + let lz4chonk_post = Instant::now(); + //let lz4_dict_chonk = SerializedTerrainChunk::from_chunk(&chunk, + // &*dictionary); + + let deflatechonk_pre = Instant::now(); + let deflate_chonk = do_deflate_flate2(&bincode::serialize(&chunk).unwrap()); + let deflatechonk_post = Instant::now(); + + let dyna: Dyna<_, _, ColumnAccess> = chonk_to_dyna(&chunk, Block::empty()); + let ser_dyna = bincode::serialize(&dyna).unwrap(); + for w in ser_dyna.windows(k) { + *histogram2.entry(w.to_vec()).or_default() += 1; + } + if x % 128 == 0 { + histogram_to_dictionary(&histogram2, &mut dictionary2); + } + let lz4_dyna = lz4_with_dictionary(&*ser_dyna, &[]); + //let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2); + let deflate_dyna = do_deflate(&*ser_dyna); + let deflate_channeled_dyna = + do_deflate_flate2(&bincode::serialize(&channelize_dyna(&dyna)).unwrap()); + let n = uncompressed.len(); + let sizes = [ + lz4_chonk.len() as f32 / n as f32, + deflate_chonk.len() as f32 / n as f32, + lz4_dyna.len() as f32 / n as f32, + deflate_dyna.len() as f32 / n as f32, + deflate_channeled_dyna.len() as f32 / n as f32, + ]; + let i = sizes + .iter() + .enumerate() + .fold((1.0, 0), |(best, i), (j, ratio)| { + if ratio < &best { + (*ratio, j) + } else { + (best, i) + } + }) + .1; + let timings = [ + (lz4chonk_post - lz4chonk_pre).subsec_nanos(), + (deflatechonk_post - deflatechonk_pre).subsec_nanos(), + ]; + trace!( + "{} {}: uncompressed: {}, {:?} {} {:?}", + x, + y, + n, + sizes, + i, + timings + ); + for i in 0..5 { + totals[i] += sizes[i]; + } + for i in 0..2 { + total_timings[i] += timings[i] as f32; + } + count += 1; + } + if x % 64 == 0 { + println!("Chunks processed: {}\n", count); + println!("Average lz4_chonk: {}", totals[0] / count as f32); + println!("Average deflate_chonk: {}", totals[1] / count as f32); + println!("Average lz4_dyna: {}", totals[2] / count as f32); + println!("Average deflate_dyna: {}", totals[3] / count as f32); + println!( + "Average deflate_channeled_dyna: {}", + totals[4] / count as f32 + ); + println!(""); + println!( + "Average lz4_chonk nanos : {:02}", + total_timings[0] / count as f32 + ); + println!( + "Average deflate_chonk nanos: {:02}", + total_timings[1] / count as f32 + ); + println!("-----"); + } + } + histogram.clear(); + } +} From 15e32e56553d019ea1b3a89ee40337dfbd77ec76 Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Tue, 20 Apr 2021 19:33:42 -0400 Subject: [PATCH 02/41] Move terrain compression code to common_net and disable redundant LZ4 compression on the terrain stream. --- Cargo.lock | 4 +-- client/src/lib.rs | 12 ++++--- common/Cargo.toml | 5 +-- common/net/Cargo.toml | 2 ++ common/net/src/msg/mod.rs | 59 ++++++++++++++++++++++++++++++++ common/net/src/msg/server.rs | 8 ++--- common/src/terrain/mod.rs | 51 --------------------------- server/src/connection_handler.rs | 2 +- server/src/sys/msg/terrain.rs | 6 ++-- server/src/sys/terrain.rs | 6 ++-- server/src/sys/terrain_sync.rs | 25 +++++++------- 11 files changed, 94 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e41c084765..c36ba62d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5463,7 +5463,6 @@ dependencies = [ "approx 0.4.0", "arraygen", "assets_manager", - "bincode", "bitflags", "criterion", "crossbeam-channel", @@ -5471,7 +5470,6 @@ dependencies = [ "csv", "dot_vox", "enum-iterator", - "flate2", "hashbrown", "image", "indexmap", @@ -5535,6 +5533,8 @@ dependencies = [ name = "veloren-common-net" version = "0.9.0" dependencies = [ + "bincode", + "flate2", "hashbrown", "serde", "specs", diff --git a/client/src/lib.rs b/client/src/lib.rs index 7a6ecf3d41..d0108121fc 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1923,15 +1923,17 @@ impl Client { fn handle_server_terrain_msg(&mut self, msg: ServerGeneral) -> Result<(), Error> { match msg { ServerGeneral::TerrainChunkUpdate { key, chunk } => { - if let Some(chunk) = chunk.ok().and_then(|c| c.to_chunk()) { + if let Some(chunk) = chunk.ok().and_then(|c| c.decompress()) { self.state.insert_chunk(key, chunk); } self.pending_chunks.remove(&key); }, - ServerGeneral::TerrainBlockUpdates(mut blocks) => { - blocks.drain().for_each(|(pos, block)| { - self.state.set_block(pos, block); - }); + ServerGeneral::TerrainBlockUpdates(blocks) => { + if let Some(mut blocks) = blocks.decompress() { + blocks.drain().for_each(|(pos, block)| { + self.state.set_block(pos, block); + }); + } }, _ => unreachable!("Not a terrain message"), } diff --git a/common/Cargo.toml b/common/Cargo.toml index 5028493a73..8c3ef5d6c9 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -9,9 +9,8 @@ no-assets = [] tracy = ["common-base/tracy"] simd = ["vek/platform_intrinsics"] bin_csv = ["csv", "structopt"] -compression = ["flate2"] -default = ["simd", "compression"] +default = ["simd"] [dependencies] @@ -24,13 +23,11 @@ serde = { version = "1.0.110", features = ["derive", "rc"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] approx = "0.4.0" arraygen = "0.1.13" -bincode = "1.3.3" crossbeam-utils = "0.8.1" bitflags = "1.2" crossbeam-channel = "0.5" enum-iterator = "0.6" lazy_static = "1.4.0" -flate2 = { version = "1.0.20", optional = true } num-derive = "0.3" num-traits = "0.2" ordered-float = { version = "2.0.1", default-features = false } diff --git a/common/net/Cargo.toml b/common/net/Cargo.toml index 209c2d6a18..0661f79697 100644 --- a/common/net/Cargo.toml +++ b/common/net/Cargo.toml @@ -14,6 +14,8 @@ default = ["simd"] common = {package = "veloren-common", path = "../../common"} #inline_tweak = "1.0.2" +bincode = "1.3.3" +flate2 = "1.0.20" sum_type = "0.2.0" vek = { version = "=0.14.1", features = ["serde"] } tracing = { version = "0.1", default-features = false } diff --git a/common/net/src/msg/mod.rs b/common/net/src/msg/mod.rs index 667038ac78..2d2421f9cc 100644 --- a/common/net/src/msg/mod.rs +++ b/common/net/src/msg/mod.rs @@ -15,6 +15,8 @@ pub use self::{ }; use common::character::CharacterId; use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; +use tracing::trace; #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub enum PresenceKind { @@ -42,3 +44,60 @@ pub fn validate_chat_msg(msg: &str) -> Result<(), ChatMsgValidationError> { Err(ChatMsgValidationError::TooLong) } } + +/// Wrapper for compressed, serialized data (for stuff that doesn't use the +/// default lz4 compression) +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CompressedData { + pub data: Vec, + compressed: bool, + _phantom: PhantomData, +} + +impl Deserialize<'a>> CompressedData { + pub fn compress(t: &T, level: u32) -> Self { + use flate2::{write::DeflateEncoder, Compression}; + use std::io::Write; + let uncompressed = bincode::serialize(t) + .expect("bincode serialization can only fail if a byte limit is set"); + + if uncompressed.len() >= 32 { + const EXPECT_MSG: &str = + "compression only fails for fallible Read/Write impls (which Vec is not)"; + + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(level)); + encoder.write_all(&*uncompressed).expect(EXPECT_MSG); + let compressed = encoder.finish().expect(EXPECT_MSG); + trace!( + "compressed {}, uncompressed {}, ratio {}", + compressed.len(), + uncompressed.len(), + compressed.len() as f32 / uncompressed.len() as f32 + ); + CompressedData { + data: compressed, + compressed: true, + _phantom: PhantomData, + } + } else { + CompressedData { + data: uncompressed, + compressed: false, + _phantom: PhantomData, + } + } + } + + pub fn decompress(&self) -> Option { + use std::io::Read; + if self.compressed { + let mut uncompressed = Vec::new(); + flate2::read::DeflateDecoder::new(&*self.data) + .read_to_end(&mut uncompressed) + .ok()?; + bincode::deserialize(&*uncompressed).ok() + } else { + bincode::deserialize(&*self.data).ok() + } + } +} diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs index 1a656e08c5..5596945f86 100644 --- a/common/net/src/msg/server.rs +++ b/common/net/src/msg/server.rs @@ -1,4 +1,4 @@ -use super::{world_msg::EconomyInfo, ClientType, EcsCompPacket, PingMsg}; +use super::{world_msg::EconomyInfo, ClientType, CompressedData, EcsCompPacket, PingMsg}; use crate::sync; use common::{ character::{self, CharacterItem}, @@ -6,7 +6,7 @@ use common::{ outcome::Outcome, recipe::RecipeBook, resources::TimeOfDay, - terrain::{Block, SerializedTerrainChunk}, + terrain::{Block, TerrainChunk}, trade::{PendingTrade, SitePrices, TradeId, TradeResult}, uid::Uid, }; @@ -106,9 +106,9 @@ pub enum ServerGeneral { // Ingame related AND terrain stream TerrainChunkUpdate { key: Vec2, - chunk: Result, + chunk: Result, ()>, }, - TerrainBlockUpdates(HashMap, Block>), + TerrainBlockUpdates(CompressedData, Block>>), // Always possible PlayerListUpdate(PlayerListUpdate), /// A message to go into the client chat box. The client is responsible for diff --git a/common/src/terrain/mod.rs b/common/src/terrain/mod.rs index f1fb3a4106..5360b6c7c8 100644 --- a/common/src/terrain/mod.rs +++ b/common/src/terrain/mod.rs @@ -17,7 +17,6 @@ pub use self::{ }; use roots::find_roots_cubic; use serde::{Deserialize, Serialize}; -use tracing::trace; use crate::{ vol::{ReadVol, RectVolSize}, @@ -143,56 +142,6 @@ impl TerrainChunkMeta { pub type TerrainChunk = chonk::Chonk; pub type TerrainGrid = VolGrid2d; -/// Wrapper for custom serialization strategies (e.g. compression) for terrain -/// chunks -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SerializedTerrainChunk(pub Vec); - -impl SerializedTerrainChunk { - pub fn from_chunk(chunk: &TerrainChunk) -> Self { - let uncompressed = bincode::serialize(chunk) - .expect("bincode serialization can only fail if a byte limit is set"); - #[cfg(feature = "compression")] - { - use flate2::{write::DeflateEncoder, Compression}; - use std::io::Write; - const EXPECT_MSG: &str = - "compression only fails for fallible Read/Write impls (which Vec is not)"; - - let mut encoder = DeflateEncoder::new(Vec::new(), Compression::new(5)); - encoder.write_all(&*uncompressed).expect(EXPECT_MSG); - let compressed = encoder.finish().expect(EXPECT_MSG); - trace!( - "compressed {}, uncompressed {}, ratio {}", - compressed.len(), - uncompressed.len(), - compressed.len() as f32 / uncompressed.len() as f32 - ); - SerializedTerrainChunk(compressed) - } - #[cfg(not(feature = "compression"))] - { - SerializedTerrainChunk(uncompressed) - } - } - - pub fn to_chunk(&self) -> Option { - #[cfg(feature = "compression")] - { - use std::io::Read; - let mut uncompressed = Vec::new(); - flate2::read::DeflateDecoder::new(&*self.0) - .read_to_end(&mut uncompressed) - .ok()?; - bincode::deserialize(&*uncompressed).ok() - } - #[cfg(not(feature = "compression"))] - { - bincode::deserialize(&self.0).ok() - } - } -} - impl TerrainGrid { /// Find a location suitable for spawning an entity near the given /// position (but in the same chunk). diff --git a/server/src/connection_handler.rs b/server/src/connection_handler.rs index 7f0f11a198..1f7bb553af 100644 --- a/server/src/connection_handler.rs +++ b/server/src/connection_handler.rs @@ -105,7 +105,7 @@ impl ConnectionHandler { let mut register_stream = participant.open(3, reliablec, 500).await?; let character_screen_stream = participant.open(3, reliablec, 500).await?; let in_game_stream = participant.open(3, reliablec, 100_000).await?; - let terrain_stream = participant.open(4, reliablec, 20_000).await?; + let terrain_stream = participant.open(4, reliable, 20_000).await?; let server_data = receiver.recv()?; diff --git a/server/src/sys/msg/terrain.rs b/server/src/sys/msg/terrain.rs index 9725910373..009d8dfa6e 100644 --- a/server/src/sys/msg/terrain.rs +++ b/server/src/sys/msg/terrain.rs @@ -2,11 +2,11 @@ use crate::{client::Client, metrics::NetworkRequestMetrics, presence::Presence}; use common::{ comp::Pos, event::{EventBus, ServerEvent}, - terrain::{SerializedTerrainChunk, TerrainChunkSize, TerrainGrid}, + terrain::{TerrainChunkSize, TerrainGrid}, vol::RectVolSize, }; use common_ecs::{Job, Origin, ParMode, Phase, System}; -use common_net::msg::{ClientGeneral, ServerGeneral}; +use common_net::msg::{ClientGeneral, CompressedData, ServerGeneral}; use rayon::iter::ParallelIterator; use specs::{Entities, Join, ParJoin, Read, ReadExpect, ReadStorage}; use std::sync::Arc; @@ -80,7 +80,7 @@ impl<'a> System<'a> for Sys { network_metrics.chunks_served_from_memory.inc(); client.send(ServerGeneral::TerrainChunkUpdate { key, - chunk: Ok(SerializedTerrainChunk::from_chunk(&chunk)), + chunk: Ok(CompressedData::compress(&chunk, 5)), })? }, None => { diff --git a/server/src/sys/terrain.rs b/server/src/sys/terrain.rs index 3f7cbb145f..50c9be9cce 100644 --- a/server/src/sys/terrain.rs +++ b/server/src/sys/terrain.rs @@ -10,11 +10,11 @@ use common::{ event::{EventBus, ServerEvent}, generation::get_npc_name, npc::NPC_NAMES, - terrain::{SerializedTerrainChunk, TerrainGrid}, + terrain::TerrainGrid, LoadoutBuilder, SkillSetBuilder, }; use common_ecs::{Job, Origin, Phase, System}; -use common_net::msg::ServerGeneral; +use common_net::msg::{CompressedData, ServerGeneral}; use common_state::TerrainChanges; use comp::Behavior; use specs::{Join, Read, ReadStorage, Write, WriteExpect}; @@ -108,7 +108,7 @@ impl<'a> System<'a> for Sys { if lazy_msg.is_none() { lazy_msg = Some(client.prepare(ServerGeneral::TerrainChunkUpdate { key, - chunk: Ok(SerializedTerrainChunk::from_chunk(&*chunk)), + chunk: Ok(CompressedData::compress(&*chunk, 5)), })); } lazy_msg.as_ref().map(|ref msg| client.send_prepared(&msg)); diff --git a/server/src/sys/terrain_sync.rs b/server/src/sys/terrain_sync.rs index 9f203874f9..ebbe7bb95c 100644 --- a/server/src/sys/terrain_sync.rs +++ b/server/src/sys/terrain_sync.rs @@ -1,10 +1,7 @@ use crate::{client::Client, presence::Presence}; -use common::{ - comp::Pos, - terrain::{SerializedTerrainChunk, TerrainGrid}, -}; +use common::{comp::Pos, terrain::TerrainGrid}; use common_ecs::{Job, Origin, Phase, System}; -use common_net::msg::ServerGeneral; +use common_net::msg::{CompressedData, ServerGeneral}; use common_state::TerrainChanges; use specs::{Join, Read, ReadExpect, ReadStorage}; use std::sync::Arc; @@ -42,7 +39,7 @@ impl<'a> System<'a> for Sys { lazy_msg = Some(client.prepare(ServerGeneral::TerrainChunkUpdate { key: *chunk_key, chunk: Ok(match terrain.get_key(*chunk_key) { - Some(chunk) => SerializedTerrainChunk::from_chunk(&chunk), + Some(chunk) => CompressedData::compress(&chunk, 5), None => break 'chunk, }), })); @@ -54,14 +51,16 @@ impl<'a> System<'a> for Sys { // TODO: Don't send all changed blocks to all clients // Sync changed blocks - let mut lazy_msg = None; - for (_, client) in (&presences, &clients).join() { - if lazy_msg.is_none() { - lazy_msg = Some(client.prepare(ServerGeneral::TerrainBlockUpdates( - terrain_changes.modified_blocks.clone(), - ))); + if !terrain_changes.modified_blocks.is_empty() { + let mut lazy_msg = None; + for (_, client) in (&presences, &clients).join() { + if lazy_msg.is_none() { + lazy_msg = Some(client.prepare(ServerGeneral::TerrainBlockUpdates( + CompressedData::compress(&terrain_changes.modified_blocks, 2), + ))); + } + lazy_msg.as_ref().map(|ref msg| client.send_prepared(&msg)); } - lazy_msg.as_ref().map(|ref msg| client.send_prepared(&msg)); } } } From 4f31f1ca3806a37cf7ea1f082e8775600582082a Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Tue, 20 Apr 2021 20:05:45 -0400 Subject: [PATCH 03/41] Switch the `chunk_compression_benchmarks` to using a spiral around the origin instead of the top-left corner to get a more representative sample. --- world/src/bin/chunk_compression_benchmarks.rs | 194 +++++++++--------- 1 file changed, 99 insertions(+), 95 deletions(-) diff --git a/world/src/bin/chunk_compression_benchmarks.rs b/world/src/bin/chunk_compression_benchmarks.rs index a20cb03e53..84c4038dcc 100644 --- a/world/src/bin/chunk_compression_benchmarks.rs +++ b/world/src/bin/chunk_compression_benchmarks.rs @@ -1,4 +1,5 @@ use common::{ + spiral::Spiral2d, terrain::{chonk::Chonk, Block, BlockKind, SpriteKind}, vol::{IntoVolIterator, RectVolSize, SizedVol, WriteVol}, volumes::dyna::{Access, ColumnAccess, Dyna}, @@ -144,103 +145,106 @@ fn main() { let mut totals = [0.0; 5]; let mut total_timings = [0.0; 2]; let mut count = 0; - for y in 1..sz.y { - for x in 1..sz.x { - let chunk = - world.generate_chunk(index.as_index_ref(), Vec2::new(x as _, y as _), || false); - if let Ok((chunk, _)) = chunk { - let uncompressed = bincode::serialize(&chunk).unwrap(); - for w in uncompressed.windows(k) { - *histogram.entry(w.to_vec()).or_default() += 1; - } - if x % 128 == 0 { - histogram_to_dictionary(&histogram, &mut dictionary); - } - let lz4chonk_pre = Instant::now(); - let lz4_chonk = lz4_with_dictionary(&bincode::serialize(&chunk).unwrap(), &[]); - let lz4chonk_post = Instant::now(); - //let lz4_dict_chonk = SerializedTerrainChunk::from_chunk(&chunk, - // &*dictionary); - - let deflatechonk_pre = Instant::now(); - let deflate_chonk = do_deflate_flate2(&bincode::serialize(&chunk).unwrap()); - let deflatechonk_post = Instant::now(); - - let dyna: Dyna<_, _, ColumnAccess> = chonk_to_dyna(&chunk, Block::empty()); - let ser_dyna = bincode::serialize(&dyna).unwrap(); - for w in ser_dyna.windows(k) { - *histogram2.entry(w.to_vec()).or_default() += 1; - } - if x % 128 == 0 { - histogram_to_dictionary(&histogram2, &mut dictionary2); - } - let lz4_dyna = lz4_with_dictionary(&*ser_dyna, &[]); - //let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2); - let deflate_dyna = do_deflate(&*ser_dyna); - let deflate_channeled_dyna = - do_deflate_flate2(&bincode::serialize(&channelize_dyna(&dyna)).unwrap()); - let n = uncompressed.len(); - let sizes = [ - lz4_chonk.len() as f32 / n as f32, - deflate_chonk.len() as f32 / n as f32, - lz4_dyna.len() as f32 / n as f32, - deflate_dyna.len() as f32 / n as f32, - deflate_channeled_dyna.len() as f32 / n as f32, - ]; - let i = sizes - .iter() - .enumerate() - .fold((1.0, 0), |(best, i), (j, ratio)| { - if ratio < &best { - (*ratio, j) - } else { - (best, i) - } - }) - .1; - let timings = [ - (lz4chonk_post - lz4chonk_pre).subsec_nanos(), - (deflatechonk_post - deflatechonk_pre).subsec_nanos(), - ]; - trace!( - "{} {}: uncompressed: {}, {:?} {} {:?}", - x, - y, - n, - sizes, - i, - timings - ); - for i in 0..5 { - totals[i] += sizes[i]; - } - for i in 0..2 { - total_timings[i] += timings[i] as f32; - } - count += 1; + for (i, (x, y)) in Spiral2d::new() + .radius(20) + .map(|v| (v.x + sz.x as i32 / 2, v.y + sz.y as i32 / 2)) + .enumerate() + { + let chunk = world.generate_chunk(index.as_index_ref(), Vec2::new(x as _, y as _), || false); + if let Ok((chunk, _)) = chunk { + let uncompressed = bincode::serialize(&chunk).unwrap(); + for w in uncompressed.windows(k) { + *histogram.entry(w.to_vec()).or_default() += 1; } - if x % 64 == 0 { - println!("Chunks processed: {}\n", count); - println!("Average lz4_chonk: {}", totals[0] / count as f32); - println!("Average deflate_chonk: {}", totals[1] / count as f32); - println!("Average lz4_dyna: {}", totals[2] / count as f32); - println!("Average deflate_dyna: {}", totals[3] / count as f32); - println!( - "Average deflate_channeled_dyna: {}", - totals[4] / count as f32 - ); - println!(""); - println!( - "Average lz4_chonk nanos : {:02}", - total_timings[0] / count as f32 - ); - println!( - "Average deflate_chonk nanos: {:02}", - total_timings[1] / count as f32 - ); - println!("-----"); + if i % 128 == 0 { + histogram_to_dictionary(&histogram, &mut dictionary); } + let lz4chonk_pre = Instant::now(); + let lz4_chonk = lz4_with_dictionary(&bincode::serialize(&chunk).unwrap(), &[]); + let lz4chonk_post = Instant::now(); + //let lz4_dict_chonk = SerializedTerrainChunk::from_chunk(&chunk, + // &*dictionary); + + let deflatechonk_pre = Instant::now(); + let deflate_chonk = do_deflate_flate2(&bincode::serialize(&chunk).unwrap()); + let deflatechonk_post = Instant::now(); + + let dyna: Dyna<_, _, ColumnAccess> = chonk_to_dyna(&chunk, Block::empty()); + let ser_dyna = bincode::serialize(&dyna).unwrap(); + for w in ser_dyna.windows(k) { + *histogram2.entry(w.to_vec()).or_default() += 1; + } + if i % 128 == 0 { + histogram_to_dictionary(&histogram2, &mut dictionary2); + } + let lz4_dyna = lz4_with_dictionary(&*ser_dyna, &[]); + //let lz4_dict_dyna = lz4_with_dictionary(&*ser_dyna, &dictionary2); + let deflate_dyna = do_deflate(&*ser_dyna); + let deflate_channeled_dyna = + do_deflate_flate2(&bincode::serialize(&channelize_dyna(&dyna)).unwrap()); + let n = uncompressed.len(); + let sizes = [ + lz4_chonk.len() as f32 / n as f32, + deflate_chonk.len() as f32 / n as f32, + lz4_dyna.len() as f32 / n as f32, + deflate_dyna.len() as f32 / n as f32, + deflate_channeled_dyna.len() as f32 / n as f32, + ]; + let best_idx = sizes + .iter() + .enumerate() + .fold((1.0, 0), |(best, i), (j, ratio)| { + if ratio < &best { + (*ratio, j) + } else { + (best, i) + } + }) + .1; + let timings = [ + (lz4chonk_post - lz4chonk_pre).subsec_nanos(), + (deflatechonk_post - deflatechonk_pre).subsec_nanos(), + ]; + trace!( + "{} {}: uncompressed: {}, {:?} {} {:?}", + x, + y, + n, + sizes, + best_idx, + timings + ); + for j in 0..5 { + totals[j] += sizes[j]; + } + for j in 0..2 { + total_timings[j] += timings[j] as f32; + } + count += 1; + } + if i % 64 == 0 { + println!("Chunks processed: {}\n", count); + println!("Average lz4_chonk: {}", totals[0] / count as f32); + println!("Average deflate_chonk: {}", totals[1] / count as f32); + println!("Average lz4_dyna: {}", totals[2] / count as f32); + println!("Average deflate_dyna: {}", totals[3] / count as f32); + println!( + "Average deflate_channeled_dyna: {}", + totals[4] / count as f32 + ); + println!(""); + println!( + "Average lz4_chonk nanos : {:02}", + total_timings[0] / count as f32 + ); + println!( + "Average deflate_chonk nanos: {:02}", + total_timings[1] / count as f32 + ); + println!("-----"); + } + if i % 256 == 0 { + histogram.clear(); } - histogram.clear(); } } From 308ad4d81e93be95c529aa0eac939ab263407329 Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Tue, 20 Apr 2021 23:27:43 -0400 Subject: [PATCH 04/41] Cleanup errors introduced in rebase. --- client/src/lib.rs | 2 +- common/net/src/msg/server.rs | 2 +- server/src/sys/msg/terrain.rs | 1 - server/src/sys/terrain.rs | 2 +- server/src/sys/terrain_sync.rs | 1 - 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/client/src/lib.rs b/client/src/lib.rs index d0108121fc..63168d9337 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1924,7 +1924,7 @@ impl Client { match msg { ServerGeneral::TerrainChunkUpdate { key, chunk } => { if let Some(chunk) = chunk.ok().and_then(|c| c.decompress()) { - self.state.insert_chunk(key, chunk); + self.state.insert_chunk(key, Arc::new(chunk)); } self.pending_chunks.remove(&key); }, diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs index 5596945f86..033ed0892e 100644 --- a/common/net/src/msg/server.rs +++ b/common/net/src/msg/server.rs @@ -12,7 +12,7 @@ use common::{ }; use hashbrown::HashMap; use serde::{Deserialize, Serialize}; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use vek::*; ///This struct contains all messages the server might send (on different diff --git a/server/src/sys/msg/terrain.rs b/server/src/sys/msg/terrain.rs index 009d8dfa6e..78209ef68b 100644 --- a/server/src/sys/msg/terrain.rs +++ b/server/src/sys/msg/terrain.rs @@ -9,7 +9,6 @@ use common_ecs::{Job, Origin, ParMode, Phase, System}; use common_net::msg::{ClientGeneral, CompressedData, ServerGeneral}; use rayon::iter::ParallelIterator; use specs::{Entities, Join, ParJoin, Read, ReadExpect, ReadStorage}; -use std::sync::Arc; use tracing::{debug, trace}; /// This system will handle new messages from clients diff --git a/server/src/sys/terrain.rs b/server/src/sys/terrain.rs index 50c9be9cce..3294c13059 100644 --- a/server/src/sys/terrain.rs +++ b/server/src/sys/terrain.rs @@ -246,7 +246,7 @@ impl<'a> System<'a> for Sys { new_chunks.into_par_iter().for_each(|(key, chunk)| { let mut msg = Some(ServerGeneral::TerrainChunkUpdate { key, - chunk: Ok(chunk), + chunk: Ok(CompressedData::compress(&*chunk, 5)), }); let mut lazy_msg = None; diff --git a/server/src/sys/terrain_sync.rs b/server/src/sys/terrain_sync.rs index ebbe7bb95c..305427457b 100644 --- a/server/src/sys/terrain_sync.rs +++ b/server/src/sys/terrain_sync.rs @@ -4,7 +4,6 @@ use common_ecs::{Job, Origin, Phase, System}; use common_net::msg::{CompressedData, ServerGeneral}; use common_state::TerrainChanges; use specs::{Join, Read, ReadExpect, ReadStorage}; -use std::sync::Arc; /// This systems sends new chunks to clients as well as changes to existing /// chunks From 95d32b40bb25011994b56f9d44c8242aee5eb60f Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Tue, 20 Apr 2021 23:48:15 -0400 Subject: [PATCH 05/41] Remove redundant terrain message per MR 2166 comment. --- server/src/sys/terrain.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/server/src/sys/terrain.rs b/server/src/sys/terrain.rs index 3294c13059..42778912e9 100644 --- a/server/src/sys/terrain.rs +++ b/server/src/sys/terrain.rs @@ -93,28 +93,6 @@ impl<'a> System<'a> for Sys { // Add to list of chunks to send to nearby players. new_chunks.push((key, Arc::clone(&chunk))); - // Send the chunk to all nearby players. - let mut lazy_msg = None; - for (presence, pos, client) in (&presences, &positions, &clients).join() { - let chunk_pos = terrain.pos_key(pos.0.map(|e| e as i32)); - // Subtract 2 from the offset before computing squared magnitude - // 1 since chunks need neighbors to be meshed - // 1 to act as a buffer if the player moves in that direction - let adjusted_dist_sqr = (chunk_pos - key) - .map(|e: i32| (e.abs() as u32).saturating_sub(2)) - .magnitude_squared(); - - if adjusted_dist_sqr <= presence.view_distance.pow(2) { - if lazy_msg.is_none() { - lazy_msg = Some(client.prepare(ServerGeneral::TerrainChunkUpdate { - key, - chunk: Ok(CompressedData::compress(&*chunk, 5)), - })); - } - lazy_msg.as_ref().map(|ref msg| client.send_prepared(&msg)); - } - } - // TODO: code duplication for chunk insertion between here and state.rs // Insert the chunk into terrain changes if terrain.insert(key, chunk).is_some() { From ed7cc12213e61019180fa64aa69cd3b6517a7291 Mon Sep 17 00:00:00 2001 From: Joshua Barretto Date: Wed, 21 Apr 2021 15:59:29 +0100 Subject: [PATCH 06/41] Made characters carry lanterns higher when possible --- common/src/comp/character_state.rs | 12 ++++++++++++ voxygen/anim/src/character/jump.rs | 10 ++++++++++ voxygen/anim/src/character/mod.rs | 25 ++++++++++++++++++++----- voxygen/anim/src/character/run.rs | 14 ++++++++++++++ voxygen/anim/src/character/sit.rs | 27 +++++++++++++++++++++++++-- voxygen/anim/src/character/sneak.rs | 9 +++++++++ voxygen/anim/src/character/stand.rs | 17 +++++++++++++++++ voxygen/anim/src/character/wield.rs | 25 +++++++++++++++++++++---- voxygen/anim/src/lib.rs | 11 ++++++++++- voxygen/src/scene/figure/mod.rs | 24 +++++++++++++++++------- voxygen/src/scene/mod.rs | 16 ++++------------ 11 files changed, 159 insertions(+), 31 deletions(-) diff --git a/common/src/comp/character_state.rs b/common/src/comp/character_state.rs index 587b3f5b1f..553384db9f 100644 --- a/common/src/comp/character_state.rs +++ b/common/src/comp/character_state.rs @@ -167,6 +167,18 @@ impl CharacterState { ) } + pub fn is_using_hands(&self) -> bool { + matches!( + self, + CharacterState::Climb(_) + | CharacterState::Equipping(_) + | CharacterState::Dance + | CharacterState::Glide + | CharacterState::GlideWield + | CharacterState::Roll(_), + ) + } + pub fn is_block(&self) -> bool { matches!(self, CharacterState::BasicBlock) } pub fn is_dodge(&self) -> bool { matches!(self, CharacterState::Roll(_)) } diff --git a/voxygen/anim/src/character/jump.rs b/voxygen/anim/src/character/jump.rs index 8d1742aaf4..7bf3948940 100644 --- a/voxygen/anim/src/character/jump.rs +++ b/voxygen/anim/src/character/jump.rs @@ -192,6 +192,16 @@ impl Animation for JumpAnimation { next.lantern.scale = Vec3::one() * 0.65; next.hold.scale = Vec3::one() * 0.0; + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new(s_a.hand.0, s_a.hand.1 + 5.0, s_a.hand.2 + 9.0); + next.hand_r.orientation = Quaternion::rotation_x(2.25) * Quaternion::rotation_z(0.9); + + next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.orientation = next.hand_r.orientation.inverse() + * Quaternion::rotation_x(slow * 0.5) + * Quaternion::rotation_y(tilt * 4.0 * slow + tilt * 3.0); + } + next.torso.position = Vec3::new(0.0, 0.0, 0.0) * s_a.scaler; next.torso.orientation = Quaternion::rotation_x(0.0); next.torso.scale = Vec3::one() / 11.0 * s_a.scaler; diff --git a/voxygen/anim/src/character/mod.rs b/voxygen/anim/src/character/mod.rs index 96fb1b01dc..a060291196 100644 --- a/voxygen/anim/src/character/mod.rs +++ b/voxygen/anim/src/character/mod.rs @@ -69,8 +69,19 @@ skeleton_impls!(struct CharacterSkeleton { control, control_l, control_r, + :: // Begin non-bone fields + holding_lantern: bool, }); +impl CharacterSkeleton { + pub fn new(holding_lantern: bool) -> Self { + Self { + holding_lantern, + ..Self::default() + } + } +} + impl Skeleton for CharacterSkeleton { type Attr = SkeletonAttr; type Body = Body; @@ -93,9 +104,14 @@ impl Skeleton for CharacterSkeleton { let control_mat = chest_mat * Mat4::::from(self.control); let control_l_mat = control_mat * Mat4::::from(self.control_l); let control_r_mat = control_mat * Mat4::::from(self.control_r); + let hand_r_mat = control_r_mat * Mat4::::from(self.hand_r); let hand_l_mat = Mat4::::from(self.hand_l); - let lantern_mat = Mat4::::from(self.lantern); + let lantern_mat = if self.holding_lantern { + hand_r_mat + } else { + shorts_mat + } * Mat4::::from(self.lantern); *(<&mut [_; Self::BONE_COUNT]>::try_from(&mut buf[0..Self::BONE_COUNT]).unwrap()) = [ make_bone(head_mat), @@ -104,7 +120,7 @@ impl Skeleton for CharacterSkeleton { make_bone(chest_mat * Mat4::::from(self.back)), make_bone(shorts_mat), make_bone(control_l_mat * hand_l_mat), - make_bone(control_r_mat * Mat4::::from(self.hand_r)), + make_bone(hand_r_mat), make_bone(torso_mat * Mat4::::from(self.foot_l)), make_bone(torso_mat * Mat4::::from(self.foot_r)), make_bone(chest_mat * Mat4::::from(self.shoulder_l)), @@ -112,12 +128,11 @@ impl Skeleton for CharacterSkeleton { make_bone(chest_mat * Mat4::::from(self.glider)), make_bone(control_l_mat * Mat4::::from(self.main)), make_bone(control_r_mat * Mat4::::from(self.second)), - make_bone(shorts_mat * lantern_mat), + make_bone(lantern_mat), // FIXME: Should this be control_l_mat? make_bone(control_mat * hand_l_mat * Mat4::::from(self.hold)), ]; - // NOTE: lantern_mat.cols.w = lantern_mat * Vec4::unit_w() - Vec3::new(-0.3, 0.1, 0.8) + (lantern_mat.cols.w / 13.0).xyz() + (lantern_mat * Vec4::new(-0.0, -0.0, -1.5, 1.0)).xyz() } } diff --git a/voxygen/anim/src/character/run.rs b/voxygen/anim/src/character/run.rs index 5d5100f8d7..3c0afdbe14 100644 --- a/voxygen/anim/src/character/run.rs +++ b/voxygen/anim/src/character/run.rs @@ -256,6 +256,20 @@ impl Animation for RunAnimation { next.lantern.scale = Vec3::one() * 0.65; next.hold.scale = Vec3::one() * 0.0; + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new( + s_a.hand.0, + s_a.hand.1 + 5.0 - impact * 0.2, + s_a.hand.2 + 11.0 + impact * -0.1, + ); + next.hand_r.orientation = Quaternion::rotation_x(2.25) * Quaternion::rotation_z(0.9); + + next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.orientation = next.hand_r.orientation.inverse() + * Quaternion::rotation_x((foothorir + 0.5) * 1.0 * speednorm) + * Quaternion::rotation_y(tilt * 4.0 * foothorir + tilt * 3.0); + } + next.torso.position = Vec3::new(0.0, 0.0, 0.0) * s_a.scaler; next.torso.scale = Vec3::one() / 11.0 * s_a.scaler; diff --git a/voxygen/anim/src/character/sit.rs b/voxygen/anim/src/character/sit.rs index 74cd609d60..4de7063908 100644 --- a/voxygen/anim/src/character/sit.rs +++ b/voxygen/anim/src/character/sit.rs @@ -29,8 +29,16 @@ impl Animation for SitAnimation { let stop = (anim_time * 3.0).min(PI / 2.0).sin(); let head_look = Vec2::new( - (global_time + anim_time / 18.0).floor().mul(7331.0).sin() * 0.25, - (global_time + anim_time / 18.0).floor().mul(1337.0).sin() * 0.125, + (global_time * 0.05 + anim_time / 15.0) + .floor() + .mul(7331.0) + .sin() + * 0.25, + (global_time * 0.05 + anim_time / 15.0) + .floor() + .mul(1337.0) + .sin() + * 0.125, ); next.head.position = Vec3::new(0.0, s_a.head.0, s_a.head.1 + slow * 0.1 + stop * -0.8); next.head.orientation = Quaternion::rotation_z(head_look.x + slow * 0.2 - slow * 0.1) @@ -79,6 +87,21 @@ impl Animation for SitAnimation { next.torso.position = Vec3::new(0.0, -0.2, stop * -0.16) * s_a.scaler; + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new( + s_a.hand.0 - head_look.x * 10.0, + s_a.hand.1 + 5.0 - head_look.y * 8.0, + s_a.hand.2 + 11.0, + ); + next.hand_r.orientation = Quaternion::rotation_x(2.25) + * Quaternion::rotation_z(0.9) + * Quaternion::rotation_y(head_look.x * 3.0) + * Quaternion::rotation_x(head_look.y * 3.0); + + next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.orientation = next.hand_r.orientation.inverse(); + } + next } } diff --git a/voxygen/anim/src/character/sneak.rs b/voxygen/anim/src/character/sneak.rs index ca0a41d99e..13897e5a0a 100644 --- a/voxygen/anim/src/character/sneak.rs +++ b/voxygen/anim/src/character/sneak.rs @@ -158,6 +158,15 @@ impl Animation for SneakAnimation { next.foot_r.position = Vec3::new(s_a.foot.0, 4.0 + s_a.foot.1, s_a.foot.2); } + + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new(s_a.hand.0, s_a.hand.1 + 5.0, s_a.hand.2 + 9.0); + next.hand_r.orientation = Quaternion::rotation_x(2.5); + + next.lantern.position = Vec3::new(0.0, 1.5, -5.5); + next.lantern.orientation = next.hand_r.orientation.inverse(); + } + next } } diff --git a/voxygen/anim/src/character/stand.rs b/voxygen/anim/src/character/stand.rs index 34f8cbcb38..c093140847 100644 --- a/voxygen/anim/src/character/stand.rs +++ b/voxygen/anim/src/character/stand.rs @@ -32,6 +32,7 @@ impl Animation for StandAnimation { let mut next = (*skeleton).clone(); let slow = (anim_time * 1.0).sin(); + let fast = (anim_time * 5.0).sin(); let impact = (avg_vel.z).max(-15.0); let head_look = Vec2::new( ((global_time + anim_time) / 10.0).floor().mul(7331.0).sin() * 0.15, @@ -142,6 +143,22 @@ impl Animation for StandAnimation { next.lantern.position = Vec3::new(s_a.lantern.0, s_a.lantern.1, s_a.lantern.2); next.lantern.orientation = Quaternion::rotation_x(0.1) * Quaternion::rotation_y(0.1); + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new( + s_a.hand.0 - head_look.x * 10.0, + s_a.hand.1 + 5.0 - head_look.y * 8.0 + slow * 0.15 - impact * 0.2, + s_a.hand.2 + 11.0 + slow * 0.5 + impact * -0.1, + ); + next.hand_r.orientation = Quaternion::rotation_x(2.25 + slow * -0.06 + impact * -0.1) + * Quaternion::rotation_z(0.9) + * Quaternion::rotation_y(head_look.x * 3.0) + * Quaternion::rotation_x(head_look.y * 3.0); + + next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.orientation = + next.hand_r.orientation.inverse() * Quaternion::rotation_x(fast * 0.1); + } + next.torso.position = Vec3::new(0.0, 0.0, 0.0) * s_a.scaler; next.second.scale = Vec3::one(); next.second.scale = match hands { diff --git a/voxygen/anim/src/character/wield.rs b/voxygen/anim/src/character/wield.rs index 1dc4244f63..4764eedfa2 100644 --- a/voxygen/anim/src/character/wield.rs +++ b/voxygen/anim/src/character/wield.rs @@ -320,15 +320,17 @@ impl Animation for WieldAnimation { }; match hands { (None, None) | (None, Some(Hands::One)) => { - next.hand_l.position = Vec3::new(-4.5, 8.0, 5.0); - next.hand_l.orientation = Quaternion::rotation_x(1.9) * Quaternion::rotation_y(-0.5) + //next.hand_l.position = Vec3::new(-4.5, 8.0, 5.0); + //next.hand_l.orientation = Quaternion::rotation_x(1.9) * + // Quaternion::rotation_y(-0.5) }, (_, _) => {}, }; match hands { (None, None) | (Some(Hands::One), None) => { - next.hand_r.position = Vec3::new(4.5, 8.0, 5.0); - next.hand_r.orientation = Quaternion::rotation_x(1.9) * Quaternion::rotation_y(0.5) + //next.hand_r.position = Vec3::new(4.5, 8.0, 5.0); + //next.hand_r.orientation = Quaternion::rotation_x(1.9) * + // Quaternion::rotation_y(0.5) }, (_, _) => {}, }; @@ -337,6 +339,21 @@ impl Animation for WieldAnimation { next.second = next.main; } + if skeleton.holding_lantern { + next.hand_r.position = Vec3::new( + s_a.hand.0 - head_look.x * 10.0, + s_a.hand.1 + 5.0 + slow * 0.15, + s_a.hand.2 + 9.0 + head_look.y * 18.0 + slow * 0.5, + ); + next.hand_r.orientation = Quaternion::rotation_x(2.25 + slow * -0.06) + * Quaternion::rotation_z(0.9) + * Quaternion::rotation_y(head_look.x * 3.0) + * Quaternion::rotation_x(head_look.y * 3.0); + + next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.orientation = next.hand_r.orientation.inverse(); + } + next } } diff --git a/voxygen/anim/src/lib.rs b/voxygen/anim/src/lib.rs index c424b4d869..8895e92841 100644 --- a/voxygen/anim/src/lib.rs +++ b/voxygen/anim/src/lib.rs @@ -5,12 +5,15 @@ compile_error!("Can't use both \"be-dyn-lib\" and \"use-dyn-lib\" features at once"); macro_rules! skeleton_impls { - { struct $Skeleton:ident { $( $(+)? $bone:ident ),* $(,)? } } => { + { struct $Skeleton:ident { $( $(+)? $bone:ident ),* $(,)? $(:: $($field:ident : $field_ty:ty),* $(,)? )? } } => { #[derive(Clone, Default)] pub struct $Skeleton { $( $bone: $crate::Bone, )* + $($( + $field : $field_ty, + )*)? } impl<'a, Factor> $crate::vek::Lerp for &'a $Skeleton @@ -25,6 +28,9 @@ macro_rules! skeleton_impls { $( $bone: Lerp::lerp_unclamped_precise(from.$bone, to.$bone, factor), )* + $($( + $field : to.$field.clone(), + )*)? } } @@ -33,6 +39,9 @@ macro_rules! skeleton_impls { $( $bone: Lerp::lerp_unclamped(from.$bone, to.$bone, factor), )* + $($( + $field : to.$field.clone(), + )*)? } } } diff --git a/voxygen/src/scene/figure/mod.rs b/voxygen/src/scene/figure/mod.rs index 5e931d8374..4f4c9c8deb 100644 --- a/voxygen/src/scene/figure/mod.rs +++ b/voxygen/src/scene/figure/mod.rs @@ -28,7 +28,7 @@ use anim::{ use common::{ comp::{ inventory::slot::EquipSlot, - item::{ItemKind, ToolKind}, + item::{Hands, ItemKind, ToolKind}, Body, CharacterState, Controller, Health, Inventory, Item, Last, LightAnimation, LightEmitter, Ori, PhysicsState, PoiseState, Pos, Scale, Vel, }, @@ -584,6 +584,7 @@ impl FigureMgr { health, inventory, item, + light_emitter, ), ) in ( &ecs.entities(), @@ -599,6 +600,7 @@ impl FigureMgr { ecs.read_storage::().maybe(), ecs.read_storage::().maybe(), ecs.read_storage::().maybe(), + ecs.read_storage::().maybe(), ) .join() .enumerate() @@ -760,12 +762,20 @@ impl FigureMgr { &slow_jobs, ); + let holding_lantern = inventory + .map_or(false, |i| i.equipped(EquipSlot::Lantern).is_some()) + && light_emitter.is_some() + && !(matches!(second_tool_hand, Some(_)) + && character.map_or(false, |c| c.is_wield())) + && !character.map_or(false, |c| c.is_using_hands()) + && physics.in_liquid().is_none(); + let state = self .states .character_states .entry(entity) .or_insert_with(|| { - FigureState::new(renderer, CharacterSkeleton::default()) + FigureState::new(renderer, CharacterSkeleton::new(holding_lantern)) }); // Average velocity relative to the current ground @@ -787,7 +797,7 @@ impl FigureMgr { ) { // Standing (true, false, false) => anim::character::StandAnimation::update_skeleton( - &CharacterSkeleton::default(), + &CharacterSkeleton::new(holding_lantern), (active_tool_kind, second_tool_kind, hands, time, rel_avg_vel), state.state_time, &mut state_animation_rate, @@ -795,7 +805,7 @@ impl FigureMgr { ), // Running (true, true, false) => anim::character::RunAnimation::update_skeleton( - &CharacterSkeleton::default(), + &CharacterSkeleton::new(holding_lantern), ( active_tool_kind, second_tool_kind, @@ -814,7 +824,7 @@ impl FigureMgr { ), // In air (false, _, false) => anim::character::JumpAnimation::update_skeleton( - &CharacterSkeleton::default(), + &CharacterSkeleton::new(holding_lantern), ( active_tool_kind, second_tool_kind, @@ -831,7 +841,7 @@ impl FigureMgr { ), // Swim (_, _, true) => anim::character::SwimAnimation::update_skeleton( - &CharacterSkeleton::default(), + &CharacterSkeleton::new(holding_lantern), ( active_tool_kind, second_tool_kind, @@ -1376,7 +1386,7 @@ impl FigureMgr { }, CharacterState::BasicBlock { .. } => { anim::character::BlockAnimation::update_skeleton( - &CharacterSkeleton::default(), + &CharacterSkeleton::new(holding_lantern), (active_tool_kind, second_tool_kind, time), state.state_time, &mut state_animation_rate, diff --git a/voxygen/src/scene/mod.rs b/voxygen/src/scene/mod.rs index eccdbf5f90..daff82a064 100644 --- a/voxygen/src/scene/mod.rs +++ b/voxygen/src/scene/mod.rs @@ -560,7 +560,6 @@ impl Scene { lights.extend( ( &scene_data.state.ecs().read_storage::(), - scene_data.state.ecs().read_storage::().maybe(), scene_data .state .ecs() @@ -572,23 +571,16 @@ impl Scene { .read_storage::(), ) .join() - .filter(|(pos, _, _, light_anim)| { + .filter(|(pos, _, light_anim)| { light_anim.col != Rgb::zero() && light_anim.strength > 0.0 && (pos.0.distance_squared(player_pos) as f32) < loaded_distance.powi(2) + LIGHT_DIST_RADIUS }) - .map(|(pos, ori, interpolated, light_anim)| { + .map(|(pos, interpolated, light_anim)| { // Use interpolated values if they are available - let (pos, rot) = interpolated - .map_or((pos.0, ori.copied().unwrap_or_default()), |i| { - (i.pos, i.ori) - }); - Light::new( - pos + (rot.to_quat() * light_anim.offset), - light_anim.col, - light_anim.strength, - ) + let pos = interpolated.map_or(pos.0, |i| i.pos); + Light::new(pos + light_anim.offset, light_anim.col, light_anim.strength) }) .chain( self.event_lights From 2c95165c82b3c9b169715408e3b497f81a94d599 Mon Sep 17 00:00:00 2001 From: Joshua Barretto Date: Wed, 21 Apr 2021 16:54:03 +0100 Subject: [PATCH 07/41] Improvements to lantern holding when sitting --- voxygen/anim/src/character/run.rs | 2 +- voxygen/anim/src/character/sit.rs | 4 ++-- voxygen/anim/src/character/stand.rs | 2 +- voxygen/anim/src/character/wield.rs | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/voxygen/anim/src/character/run.rs b/voxygen/anim/src/character/run.rs index 3c0afdbe14..25d9139715 100644 --- a/voxygen/anim/src/character/run.rs +++ b/voxygen/anim/src/character/run.rs @@ -264,7 +264,7 @@ impl Animation for RunAnimation { ); next.hand_r.orientation = Quaternion::rotation_x(2.25) * Quaternion::rotation_z(0.9); - next.lantern.position = Vec3::new(0.0, 0.0, -3.5); + next.lantern.position = Vec3::new(0.0, 0.0, -2.5); next.lantern.orientation = next.hand_r.orientation.inverse() * Quaternion::rotation_x((foothorir + 0.5) * 1.0 * speednorm) * Quaternion::rotation_y(tilt * 4.0 * foothorir + tilt * 3.0); diff --git a/voxygen/anim/src/character/sit.rs b/voxygen/anim/src/character/sit.rs index 4de7063908..558e184ba6 100644 --- a/voxygen/anim/src/character/sit.rs +++ b/voxygen/anim/src/character/sit.rs @@ -90,8 +90,8 @@ impl Animation for SitAnimation { if skeleton.holding_lantern { next.hand_r.position = Vec3::new( s_a.hand.0 - head_look.x * 10.0, - s_a.hand.1 + 5.0 - head_look.y * 8.0, - s_a.hand.2 + 11.0, + s_a.hand.1 + 5.0 + head_look.y * 12.0, + s_a.hand.2 + 9.0 - head_look.y * 12.0, ); next.hand_r.orientation = Quaternion::rotation_x(2.25) * Quaternion::rotation_z(0.9) diff --git a/voxygen/anim/src/character/stand.rs b/voxygen/anim/src/character/stand.rs index c093140847..242f365de4 100644 --- a/voxygen/anim/src/character/stand.rs +++ b/voxygen/anim/src/character/stand.rs @@ -149,7 +149,7 @@ impl Animation for StandAnimation { s_a.hand.1 + 5.0 - head_look.y * 8.0 + slow * 0.15 - impact * 0.2, s_a.hand.2 + 11.0 + slow * 0.5 + impact * -0.1, ); - next.hand_r.orientation = Quaternion::rotation_x(2.25 + slow * -0.06 + impact * -0.1) + next.hand_r.orientation = Quaternion::rotation_x(2.5 + slow * -0.06 + impact * -0.1) * Quaternion::rotation_z(0.9) * Quaternion::rotation_y(head_look.x * 3.0) * Quaternion::rotation_x(head_look.y * 3.0); diff --git a/voxygen/anim/src/character/wield.rs b/voxygen/anim/src/character/wield.rs index 4764eedfa2..aaa4f035d2 100644 --- a/voxygen/anim/src/character/wield.rs +++ b/voxygen/anim/src/character/wield.rs @@ -320,17 +320,17 @@ impl Animation for WieldAnimation { }; match hands { (None, None) | (None, Some(Hands::One)) => { - //next.hand_l.position = Vec3::new(-4.5, 8.0, 5.0); - //next.hand_l.orientation = Quaternion::rotation_x(1.9) * - // Quaternion::rotation_y(-0.5) + next.hand_l.position = Vec3::new(-8.0, 2.0, 1.0); + next.hand_l.orientation = + Quaternion::rotation_x(0.5) * Quaternion::rotation_y(0.25); }, (_, _) => {}, }; match hands { (None, None) | (Some(Hands::One), None) => { - //next.hand_r.position = Vec3::new(4.5, 8.0, 5.0); - //next.hand_r.orientation = Quaternion::rotation_x(1.9) * - // Quaternion::rotation_y(0.5) + next.hand_r.position = Vec3::new(8.0, 2.0, 1.0); + next.hand_r.orientation = + Quaternion::rotation_x(0.5) * Quaternion::rotation_y(-0.25); }, (_, _) => {}, }; From 81ba200e4848ad5c53b9d0b2eeffbfc219748735 Mon Sep 17 00:00:00 2001 From: Joshua Barretto Date: Wed, 21 Apr 2021 17:10:53 +0100 Subject: [PATCH 08/41] Improve dynamic light reflection for hill climbing --- assets/voxygen/shaders/include/light.glsl | 3 ++- voxygen/src/scene/figure/mod.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/assets/voxygen/shaders/include/light.glsl b/assets/voxygen/shaders/include/light.glsl index e30c1fc052..8c64bd037d 100644 --- a/assets/voxygen/shaders/include/light.glsl +++ b/assets/voxygen/shaders/include/light.glsl @@ -184,7 +184,8 @@ float lights_at(vec3 wpos, vec3 wnorm, vec3 /*cam_to_frag*/view_dir, vec3 mu, ve #if (LIGHTING_TYPE & LIGHTING_TYPE_TRANSMISSION) != 0 is_direct = true; #endif - vec3 direct_light = PI * color * strength * square_factor * light_reflection_factor(/*direct_norm_dir*/wnorm, /*cam_to_frag*/view_dir, direct_light_dir, k_d, k_s, alpha, voxel_norm, voxel_lighting); + vec3 lrf = light_reflection_factor(/*direct_norm_dir*/wnorm, /*cam_to_frag*/view_dir, direct_light_dir, k_d, k_s, alpha, voxel_norm, voxel_lighting); + vec3 direct_light = PI * color * strength * square_factor * pow(lrf, vec3(0.5)); // TODO: Don't use ^0.5, it's non-physical but helps with hill climbing float computed_shadow = ShadowCalculationPoint(i, -difference, wnorm, wpos/*, light_distance*/); // directed_light += is_direct ? max(computed_shadow, /*LIGHT_AMBIANCE*/0.0) * direct_light * square_factor : vec3(0.0); #ifdef FIGURE_SHADER diff --git a/voxygen/src/scene/figure/mod.rs b/voxygen/src/scene/figure/mod.rs index 4f4c9c8deb..c048dc16c1 100644 --- a/voxygen/src/scene/figure/mod.rs +++ b/voxygen/src/scene/figure/mod.rs @@ -28,7 +28,7 @@ use anim::{ use common::{ comp::{ inventory::slot::EquipSlot, - item::{Hands, ItemKind, ToolKind}, + item::{ItemKind, ToolKind}, Body, CharacterState, Controller, Health, Inventory, Item, Last, LightAnimation, LightEmitter, Ori, PhysicsState, PoiseState, Pos, Scale, Vel, }, From 37713cb4ade24436a347f96310c6daf9b02ffb73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludvig=20B=C3=B6klin?= Date: Wed, 21 Apr 2021 18:26:23 +0200 Subject: [PATCH 09/41] Fix ArrowTurret drag coefficient --- common/src/comp/fluid_dynamics.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/comp/fluid_dynamics.rs b/common/src/comp/fluid_dynamics.rs index 6e5a688240..79b60eaf2e 100644 --- a/common/src/comp/fluid_dynamics.rs +++ b/common/src/comp/fluid_dynamics.rs @@ -149,6 +149,7 @@ impl Body { // very streamlined objects object::Body::Arrow | object::Body::ArrowSnake + | object::Body::ArrowTurret | object::Body::FireworkBlue | object::Body::FireworkGreen | object::Body::FireworkPurple From 1af4a042311e38b4488eb895e69457add732dc01 Mon Sep 17 00:00:00 2001 From: Imbris Date: Wed, 21 Apr 2021 17:10:13 +0000 Subject: [PATCH 10/41] Revert "Merge branch 'revert-b10718c5' into 'master'" This reverts merge request !2172 --- CHANGELOG.md | 2 + Cargo.lock | 8 +- Cargo.toml | 2 +- client/Cargo.toml | 5 +- client/src/lib.rs | 4 +- common/net/src/msg/server.rs | 4 +- common/src/cached_spatial_grid.rs | 21 + common/src/lib.rs | 4 + common/src/util/mod.rs | 10 +- .../src/phys => src/util}/spatial_grid.rs | 35 +- common/state/src/build_areas.rs | 54 ++ common/state/src/lib.rs | 596 +----------------- common/state/src/state.rs | 530 ++++++++++++++++ common/{sys => systems}/Cargo.toml | 2 +- common/{sys => systems}/src/aura.rs | 0 common/{sys => systems}/src/beam.rs | 0 common/{sys => systems}/src/buff.rs | 0 .../src/character_behavior.rs | 0 common/{sys => systems}/src/controller.rs | 0 common/{sys => systems}/src/interpolation.rs | 0 common/{sys => systems}/src/lib.rs | 2 +- common/{sys => systems}/src/melee.rs | 0 common/{sys => systems}/src/mount.rs | 0 common/{sys => systems}/src/phys.rs | 96 +-- common/{sys => systems}/src/projectile.rs | 0 common/{sys => systems}/src/shockwave.rs | 0 common/{sys => systems}/src/stats.rs | 0 server/Cargo.toml | 4 +- server/src/lib.rs | 2 +- server/src/sys/agent.rs | 41 +- server/src/sys/mod.rs | 2 +- server/src/sys/msg/terrain.rs | 5 +- server/src/sys/terrain.rs | 58 +- server/src/sys/terrain_sync.rs | 7 +- voxygen/Cargo.toml | 4 +- voxygen/src/ecs/sys.rs | 2 +- 36 files changed, 803 insertions(+), 697 deletions(-) create mode 100644 common/src/cached_spatial_grid.rs rename common/{sys/src/phys => src/util}/spatial_grid.rs (74%) create mode 100644 common/state/src/build_areas.rs create mode 100644 common/state/src/state.rs rename common/{sys => systems}/Cargo.toml (97%) rename common/{sys => systems}/src/aura.rs (100%) rename common/{sys => systems}/src/beam.rs (100%) rename common/{sys => systems}/src/buff.rs (100%) rename common/{sys => systems}/src/character_behavior.rs (100%) rename common/{sys => systems}/src/controller.rs (100%) rename common/{sys => systems}/src/interpolation.rs (100%) rename common/{sys => systems}/src/lib.rs (98%) rename common/{sys => systems}/src/melee.rs (100%) rename common/{sys => systems}/src/mount.rs (100%) rename common/{sys => systems}/src/phys.rs (96%) rename common/{sys => systems}/src/projectile.rs (100%) rename common/{sys => systems}/src/shockwave.rs (100%) rename common/{sys => systems}/src/stats.rs (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e43717338f..cb78c2f892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Jump has been decreased in height but extended in length as a result of the new gravity - Fall damage has been adjusted with the new gravity in mind - Projectiles now generally have a different arc because they no longer have their own gravity modifier +- Increased agent system target search efficiency speeding up the server +- Added more parallelization to terrain serialization and removed extra cloning speeding up the server ### Removed diff --git a/Cargo.lock b/Cargo.lock index b27aa7160e..ef8a66c119 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5443,7 +5443,7 @@ dependencies = [ "veloren-common-frontend", "veloren-common-net", "veloren-common-state", - "veloren-common-sys", + "veloren-common-systems", "veloren-network", ] @@ -5557,7 +5557,7 @@ dependencies = [ ] [[package]] -name = "veloren-common-sys" +name = "veloren-common-systems" version = "0.9.0" dependencies = [ "hashbrown", @@ -5682,7 +5682,7 @@ dependencies = [ "veloren-common-ecs", "veloren-common-net", "veloren-common-state", - "veloren-common-sys", + "veloren-common-systems", "veloren-network", "veloren-plugin-api", "veloren-world", @@ -5770,7 +5770,7 @@ dependencies = [ "veloren-common-frontend", "veloren-common-net", "veloren-common-state", - "veloren-common-sys", + "veloren-common-systems", "veloren-server", "veloren-voxygen-anim", "veloren-world", diff --git a/Cargo.toml b/Cargo.toml index 2eb30662d7..d1cfa92648 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ "common/ecs", "common/net", "common/state", - "common/sys", + "common/systems", "common/frontend", "client", "plugin/api", diff --git a/client/Cargo.toml b/client/Cargo.toml index 8f7619c628..e0113e12f2 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Joshua Barretto "] edition = "2018" [features] -tracy = ["common/tracy", "common-base/tracy", "common-sys/tracy", "common-net/tracy", "common-frontend/tracy"] +tracy = ["common/tracy", "common-base/tracy", "common-state/tracy", "common-systems/tracy", "common-net/tracy", "common-frontend/tracy"] simd = ["vek/platform_intrinsics"] plugins = ["common-state/plugins"] bin_bot = ["common-ecs", "serde", "ron", "clap", "rustyline", "common-frontend", "async-channel"] @@ -16,7 +16,7 @@ default = ["simd"] common = { package = "veloren-common", path = "../common", features = ["no-assets"] } common-base = { package = "veloren-common-base", path = "../common/base" } common-state = { package = "veloren-common-state", path = "../common/state", default-features = false } -common-sys = { package = "veloren-common-sys", path = "../common/sys", default-features = false } +common-systems = { package = "veloren-common-systems", path = "../common/systems", default-features = false } common-net = { package = "veloren-common-net", path = "../common/net" } network = { package = "veloren-network", path = "../network", features = ["compression"], default-features = false } @@ -32,6 +32,7 @@ vek = { version = "=0.14.1", features = ["serde"] } hashbrown = { version = "0.9", features = ["rayon", "serde", "nightly"] } authc = { git = "https://gitlab.com/veloren/auth.git", rev = "fb3dcbc4962b367253f8f2f92760ef44d2679c9a" } +#TODO: put bot in a different crate #bot only async-channel = { version = "1.6", optional = true } common-ecs = { package = "veloren-common-ecs", path = "../common/ecs", optional = true } diff --git a/client/src/lib.rs b/client/src/lib.rs index 5f225758bc..39ce8d113e 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -55,7 +55,7 @@ use common_net::{ sync::WorldSyncExt, }; use common_state::State; -use common_sys::add_local_systems; +use common_systems::add_local_systems; use comp::BuffKind; use futures_util::FutureExt; use hashbrown::{HashMap, HashSet}; @@ -1924,7 +1924,7 @@ impl Client { match msg { ServerGeneral::TerrainChunkUpdate { key, chunk } => { if let Ok(chunk) = chunk { - self.state.insert_chunk(key, *chunk); + self.state.insert_chunk(key, chunk); } self.pending_chunks.remove(&key); }, diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs index 9eabad35ca..239d3e10d0 100644 --- a/common/net/src/msg/server.rs +++ b/common/net/src/msg/server.rs @@ -12,7 +12,7 @@ use common::{ }; use hashbrown::HashMap; use serde::{Deserialize, Serialize}; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use vek::*; ///This struct contains all messages the server might send (on different @@ -106,7 +106,7 @@ pub enum ServerGeneral { // Ingame related AND terrain stream TerrainChunkUpdate { key: Vec2, - chunk: Result, ()>, + chunk: Result, ()>, }, TerrainBlockUpdates(HashMap, Block>), // Always possible diff --git a/common/src/cached_spatial_grid.rs b/common/src/cached_spatial_grid.rs new file mode 100644 index 0000000000..92b453868f --- /dev/null +++ b/common/src/cached_spatial_grid.rs @@ -0,0 +1,21 @@ +use crate::util::SpatialGrid; + +/// Cached [`SpatialGrid`] for reuse within different ecs systems during a tick. +/// This is used to accelerate queries on entities within a specific area. +/// Updated within the physics system [`crate::sys::phys::Sys`] after new entity +/// positions are calculated for the tick. So any position modifications outside +/// the physics system will not be reflected here until the next tick when the +/// physics system runs. +pub struct CachedSpatialGrid(pub SpatialGrid); + +impl Default for CachedSpatialGrid { + fn default() -> Self { + let lg2_cell_size = 5; // 32 + let lg2_large_cell_size = 6; // 64 + let radius_cutoff = 8; + + let spatial_grid = SpatialGrid::new(lg2_cell_size, lg2_large_cell_size, radius_cutoff); + + Self(spatial_grid) + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index c78e861453..d4d8f6a094 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -26,6 +26,8 @@ pub use uuid; pub mod assets; #[cfg(not(target_arch = "wasm32"))] pub mod astar; #[cfg(not(target_arch = "wasm32"))] +mod cached_spatial_grid; +#[cfg(not(target_arch = "wasm32"))] pub mod character; #[cfg(not(target_arch = "wasm32"))] pub mod clock; #[cfg(not(target_arch = "wasm32"))] pub mod cmd; @@ -78,6 +80,8 @@ pub mod uid; #[cfg(not(target_arch = "wasm32"))] pub mod volumes; +#[cfg(not(target_arch = "wasm32"))] +pub use cached_spatial_grid::CachedSpatialGrid; pub use combat::DamageSource; #[cfg(not(target_arch = "wasm32"))] pub use combat::{Damage, GroupTarget, Knockback, KnockbackDir}; diff --git a/common/src/util/mod.rs b/common/src/util/mod.rs index 2dbbc44a4f..bfad8f01e9 100644 --- a/common/src/util/mod.rs +++ b/common/src/util/mod.rs @@ -4,6 +4,9 @@ pub mod find_dist; mod option; pub mod plane; pub mod projection; +/// Contains [`SpatialGrid`] which is useful for accelerating queries of nearby +/// entities +mod spatial_grid; pub const GIT_VERSION: &str = include_str!(concat!(env!("OUT_DIR"), "/githash")); pub const GIT_TAG: &str = include_str!(concat!(env!("OUT_DIR"), "/gittag")); @@ -28,6 +31,7 @@ lazy_static::lazy_static! { pub use color::*; pub use dir::*; -pub use option::*; -pub use plane::*; -pub use projection::*; +pub use option::either_with; +pub use plane::Plane; +pub use projection::Projection; +pub use spatial_grid::SpatialGrid; diff --git a/common/sys/src/phys/spatial_grid.rs b/common/src/util/spatial_grid.rs similarity index 74% rename from common/sys/src/phys/spatial_grid.rs rename to common/src/util/spatial_grid.rs index 0e5cfd716a..aab6b67381 100644 --- a/common/sys/src/phys/spatial_grid.rs +++ b/common/src/util/spatial_grid.rs @@ -1,5 +1,6 @@ use vek::*; +#[derive(Debug)] pub struct SpatialGrid { // Uses two scales of grids so that we can have a hard limit on how far to search in the // smaller grid @@ -51,8 +52,6 @@ impl SpatialGrid { /// provided axis aligned bounding region /// NOTE: for best optimization of the iterator use `for_each` rather than a /// for loop - // TODO: a circle would be tighter (how efficient would it be to query the cells - // intersecting a circle?) pub fn in_aabr<'a>(&'a self, aabr: Aabr) -> impl Iterator + 'a { let iter = |max_entity_radius, grid: &'a hashbrown::HashMap<_, _>, lg2_cell_size| { // Add buffer for other entity radius @@ -74,4 +73,36 @@ impl SpatialGrid { self.lg2_large_cell_size, )) } + + /// Get an iterator over the entities overlapping the + /// axis aligned bounding region that contains the provided circle + /// NOTE: for best optimization of the iterator use `for_each` rather than a + /// for loop + // TODO: using the circle directly would be tighter (how efficient would it be + // to query the cells intersecting a circle?) (note: if doing this rename + // the function) + pub fn in_circle_aabr( + &self, + center: Vec2, + radius: f32, + ) -> impl Iterator + '_ { + let center = center.map(|e| e as i32); + let radius = radius.ceil() as i32; + // From conversion of center above + const CENTER_TRUNCATION_ERROR: i32 = 1; + let max_dist = radius + CENTER_TRUNCATION_ERROR; + + let aabr = Aabr { + min: center - max_dist, + max: center + max_dist, + }; + + self.in_aabr(aabr) + } + + pub fn clear(&mut self) { + self.grid.clear(); + self.large_grid.clear(); + self.largest_large_radius = self.radius_cutoff; + } } diff --git a/common/state/src/build_areas.rs b/common/state/src/build_areas.rs new file mode 100644 index 0000000000..8d1e273e77 --- /dev/null +++ b/common/state/src/build_areas.rs @@ -0,0 +1,54 @@ +use common::depot::{Depot, Id}; +use hashbrown::{hash_map, HashMap}; +use vek::*; + +/// NOTE: Please don't add `Deserialize` without checking to make sure we +/// can guarantee the invariant that every entry in `area_names` points to a +/// valid id in `areas`. +#[derive(Default)] +pub struct BuildAreas { + areas: Depot>, + area_names: HashMap>>, +} + +pub enum BuildAreaError { + /// This build area name is reserved by the system. + Reserved, + /// The build area name was not found. + NotFound, +} + +/// Build area names that can only be inserted, not removed. +const RESERVED_BUILD_AREA_NAMES: &[&str] = &["world"]; + +impl BuildAreas { + pub fn areas(&self) -> &Depot> { &self.areas } + + pub fn area_names(&self) -> &HashMap>> { &self.area_names } + + /// If the area_name is already in the map, returns Err(area_name). + pub fn insert(&mut self, area_name: String, area: Aabb) -> Result>, String> { + let area_name_entry = match self.area_names.entry(area_name) { + hash_map::Entry::Occupied(o) => return Err(o.replace_key()), + hash_map::Entry::Vacant(v) => v, + }; + let bb_id = self.areas.insert(area.made_valid()); + area_name_entry.insert(bb_id); + Ok(bb_id) + } + + pub fn remove(&mut self, area_name: &str) -> Result, BuildAreaError> { + if RESERVED_BUILD_AREA_NAMES.contains(&area_name) { + return Err(BuildAreaError::Reserved); + } + let bb_id = self + .area_names + .remove(area_name) + .ok_or(BuildAreaError::NotFound)?; + let area = self.areas.remove(bb_id).expect( + "Entries in `areas` are added before entries in `area_names` in `insert`, and that is \ + the only exposed way to add elements to `area_names`.", + ); + Ok(area) + } +} diff --git a/common/state/src/lib.rs b/common/state/src/lib.rs index c633e60771..d0047fe6de 100644 --- a/common/state/src/lib.rs +++ b/common/state/src/lib.rs @@ -1,589 +1,9 @@ +//! This crate contains the [`State`] and shared between +//! server (`veloren-server`) and the client (`veloren-client`) + +mod build_areas; #[cfg(feature = "plugins")] pub mod plugin; - -#[cfg(feature = "plugins")] -use crate::plugin::memory_manager::EcsWorld; -#[cfg(feature = "plugins")] -use crate::plugin::PluginMgr; -#[cfg(feature = "plugins")] -use common::uid::UidAllocator; -use common::{ - comp, - depot::{Depot, Id}, - event::{EventBus, LocalEvent, ServerEvent}, - outcome::Outcome, - region::RegionMap, - resources::{DeltaTime, GameMode, PlayerEntity, PlayerPhysicsSettings, Time, TimeOfDay}, - slowjob::SlowJobPool, - terrain::{Block, TerrainChunk, TerrainGrid}, - time::DayPeriod, - trade::Trades, - vol::{ReadVol, WriteVol}, -}; -use common_base::span; -use common_ecs::{PhysicsMetrics, SysMetrics}; -use common_net::sync::{interpolation as sync_interp, WorldSyncExt}; -use core::{convert::identity, time::Duration}; -use hashbrown::{hash_map, HashMap, HashSet}; -use rayon::{ThreadPool, ThreadPoolBuilder}; -use specs::{ - prelude::Resource, - shred::{Fetch, FetchMut}, - storage::{MaskedStorage as EcsMaskedStorage, Storage as EcsStorage}, - Component, DispatcherBuilder, Entity as EcsEntity, WorldExt, -}; -use std::sync::Arc; -use vek::*; - -/// How much faster should an in-game day be compared to a real day? -// TODO: Don't hard-code this. -const DAY_CYCLE_FACTOR: f64 = 24.0 * 2.0; - -/// At what point should we stop speeding up physics to compensate for lag? If -/// we speed physics up too fast, we'd skip important physics events like -/// collisions. This constant determines the upper limit. If delta time exceeds -/// this value, the game's physics will begin to produce time lag. Ideally, we'd -/// avoid such a situation. -const MAX_DELTA_TIME: f32 = 1.0; - -/// NOTE: Please don't add `Deserialize` without checking to make sure we -/// can guarantee the invariant that every entry in `area_names` points to a -/// valid id in `areas`. -#[derive(Default)] -pub struct BuildAreas { - areas: Depot>, - area_names: HashMap>>, -} - -pub enum BuildAreaError { - /// This build area name is reserved by the system. - Reserved, - /// The build area name was not found. - NotFound, -} - -/// Build area names that can only be inserted, not removed. -const RESERVED_BUILD_AREA_NAMES: &[&str] = &["world"]; - -impl BuildAreas { - pub fn areas(&self) -> &Depot> { &self.areas } - - pub fn area_names(&self) -> &HashMap>> { &self.area_names } - - /// If the area_name is already in the map, returns Err(area_name). - pub fn insert(&mut self, area_name: String, area: Aabb) -> Result>, String> { - let area_name_entry = match self.area_names.entry(area_name) { - hash_map::Entry::Occupied(o) => return Err(o.replace_key()), - hash_map::Entry::Vacant(v) => v, - }; - let bb_id = self.areas.insert(area.made_valid()); - area_name_entry.insert(bb_id); - Ok(bb_id) - } - - pub fn remove(&mut self, area_name: &str) -> Result, BuildAreaError> { - if RESERVED_BUILD_AREA_NAMES.contains(&area_name) { - return Err(BuildAreaError::Reserved); - } - let bb_id = self - .area_names - .remove(area_name) - .ok_or(BuildAreaError::NotFound)?; - let area = self.areas.remove(bb_id).expect( - "Entries in `areas` are added before entries in `area_names` in `insert`, and that is \ - the only exposed way to add elements to `area_names`.", - ); - Ok(area) - } -} - -#[derive(Default)] -pub struct BlockChange { - blocks: HashMap, Block>, -} - -impl BlockChange { - pub fn set(&mut self, pos: Vec3, block: Block) { self.blocks.insert(pos, block); } - - pub fn try_set(&mut self, pos: Vec3, block: Block) -> Option<()> { - if !self.blocks.contains_key(&pos) { - self.blocks.insert(pos, block); - Some(()) - } else { - None - } - } - - pub fn clear(&mut self) { self.blocks.clear(); } -} - -#[derive(Default)] -pub struct TerrainChanges { - pub new_chunks: HashSet>, - pub modified_chunks: HashSet>, - pub removed_chunks: HashSet>, - pub modified_blocks: HashMap, Block>, -} - -impl TerrainChanges { - pub fn clear(&mut self) { - self.new_chunks.clear(); - self.modified_chunks.clear(); - self.removed_chunks.clear(); - } -} - -#[derive(Copy, Clone)] -pub enum ExecMode { - Server, - Client, - Singleplayer, -} - -/// A type used to represent game state stored on both the client and the -/// server. This includes things like entity components, terrain data, and -/// global states like weather, time of day, etc. -pub struct State { - ecs: specs::World, - // Avoid lifetime annotation by storing a thread pool instead of the whole dispatcher - thread_pool: Arc, -} - -impl State { - /// Create a new `State` in client mode. - pub fn client() -> Self { Self::new(GameMode::Client) } - - /// Create a new `State` in server mode. - pub fn server() -> Self { Self::new(GameMode::Server) } - - pub fn new(game_mode: GameMode) -> Self { - let thread_name_infix = match game_mode { - GameMode::Server => "s", - GameMode::Client => "c", - GameMode::Singleplayer => "sp", - }; - - let thread_pool = Arc::new( - ThreadPoolBuilder::new() - .thread_name(move |i| format!("rayon-{}-{}", thread_name_infix, i)) - .build() - .unwrap(), - ); - Self { - ecs: Self::setup_ecs_world(game_mode, &thread_pool), - thread_pool, - } - } - - /// Creates ecs world and registers all the common components and resources - // TODO: Split up registering into server and client (e.g. move - // EventBus to the server) - fn setup_ecs_world(game_mode: GameMode, thread_pool: &Arc) -> specs::World { - let mut ecs = specs::World::new(); - // Uids for sync - ecs.register_sync_marker(); - // Register server -> all clients synced components. - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - - // Register components send from clients -> server - ecs.register::(); - - // Register components send directly from server -> all but one client - ecs.register::(); - - // Register components synced from client -> server -> all other clients - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - - // Register common unsynced components - ecs.register::(); - ecs.register::(); - - // Register client-local components - // TODO: only register on the client - ecs.register::(); - ecs.register::>(); - ecs.register::>(); - ecs.register::>(); - - // Register server-local components - // TODO: only register on the server - ecs.register::>(); - ecs.register::>(); - ecs.register::>(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - ecs.register::(); - - // Register synced resources used by the ECS. - ecs.insert(TimeOfDay(0.0)); - - // Register unsynced resources used by the ECS. - ecs.insert(Time(0.0)); - ecs.insert(DeltaTime(0.0)); - ecs.insert(PlayerEntity(None)); - ecs.insert(TerrainGrid::new().unwrap()); - ecs.insert(BlockChange::default()); - ecs.insert(BuildAreas::default()); - ecs.insert(TerrainChanges::default()); - ecs.insert(EventBus::::default()); - ecs.insert(game_mode); - ecs.insert(Vec::::new()); - - let slow_limit = thread_pool.current_num_threads().max(2) as u64; - let slow_limit = slow_limit / 2 + slow_limit / 4; - tracing::trace!(?slow_limit, "Slow Thread limit"); - ecs.insert(SlowJobPool::new(slow_limit, Arc::clone(&thread_pool))); - - // TODO: only register on the server - ecs.insert(EventBus::::default()); - ecs.insert(comp::group::GroupManager::default()); - ecs.insert(RegionMap::new()); - ecs.insert(SysMetrics::default()); - ecs.insert(PhysicsMetrics::default()); - ecs.insert(Trades::default()); - ecs.insert(PlayerPhysicsSettings::default()); - - // Load plugins from asset directory - #[cfg(feature = "plugins")] - ecs.insert(match PluginMgr::from_assets() { - Ok(plugin_mgr) => { - let ecs_world = EcsWorld { - entities: &ecs.entities(), - health: ecs.read_component().into(), - uid: ecs.read_component().into(), - uid_allocator: &ecs.read_resource::().into(), - player: ecs.read_component().into(), - }; - if let Err(e) = plugin_mgr - .execute_event(&ecs_world, &plugin_api::event::PluginLoadEvent { - game_mode, - }) - { - tracing::debug!(?e, "Failed to run plugin init"); - tracing::info!("Plugins disabled, enable debug logging for more information."); - PluginMgr::default() - } else { - plugin_mgr - } - }, - Err(e) => { - tracing::debug!(?e, "Failed to read plugins from assets"); - tracing::info!("Plugins disabled, enable debug logging for more information."); - PluginMgr::default() - }, - }); - - ecs - } - - /// Register a component with the state's ECS. - pub fn with_component(mut self) -> Self - where - ::Storage: Default, - { - self.ecs.register::(); - self - } - - /// Write a component attributed to a particular entity, ignoring errors. - /// - /// This should be used *only* when we can guarantee that the rest of the - /// code does not rely on the insert having succeeded (meaning the - /// entity is no longer alive!). - /// - /// Returns None if the entity was dead or there was no previous entry for - /// this component; otherwise, returns Some(old_component). - pub fn write_component_ignore_entity_dead( - &mut self, - entity: EcsEntity, - comp: C, - ) -> Option { - self.ecs - .write_storage() - .insert(entity, comp) - .ok() - .and_then(identity) - } - - /// Delete a component attributed to a particular entity. - pub fn delete_component(&mut self, entity: EcsEntity) -> Option { - self.ecs.write_storage().remove(entity) - } - - /// Read a component attributed to a particular entity. - pub fn read_component_cloned(&self, entity: EcsEntity) -> Option { - self.ecs.read_storage().get(entity).cloned() - } - - /// Read a component attributed to a particular entity. - pub fn read_component_copied(&self, entity: EcsEntity) -> Option { - self.ecs.read_storage().get(entity).copied() - } - - /// Given mutable access to the resource R, assuming the resource - /// component exists (this is already the behavior of functions like `fetch` - /// and `write_component_ignore_entity_dead`). Since all of our resources - /// are generated up front, any failure here is definitely a code bug. - pub fn mut_resource(&mut self) -> &mut R { - self.ecs.get_mut::().expect( - "Tried to fetch an invalid resource even though all our resources should be known at \ - compile time.", - ) - } - - /// Get a read-only reference to the storage of a particular component type. - pub fn read_storage(&self) -> EcsStorage>> { - self.ecs.read_storage::() - } - - /// Get a reference to the internal ECS world. - pub fn ecs(&self) -> &specs::World { &self.ecs } - - /// Get a mutable reference to the internal ECS world. - pub fn ecs_mut(&mut self) -> &mut specs::World { &mut self.ecs } - - /// Get a reference to the `TerrainChanges` structure of the state. This - /// contains information about terrain state that has changed since the - /// last game tick. - pub fn terrain_changes(&self) -> Fetch { self.ecs.read_resource() } - - /// Get the current in-game time of day. - /// - /// Note that this should not be used for physics, animations or other such - /// localised timings. - pub fn get_time_of_day(&self) -> f64 { self.ecs.read_resource::().0 } - - /// Get the current in-game day period (period of the day/night cycle) - /// Get the current in-game day period (period of the day/night cycle) - pub fn get_day_period(&self) -> DayPeriod { self.get_time_of_day().into() } - - /// Get the current in-game time. - /// - /// Note that this does not correspond to the time of day. - pub fn get_time(&self) -> f64 { self.ecs.read_resource::