From 024b6c52ef747d99a1e657bc1244bb1275e57a1d Mon Sep 17 00:00:00 2001 From: Avi Weinstock Date: Mon, 28 Jun 2021 15:02:57 -0400 Subject: [PATCH] Add block statistics generator to world/examples. --- Cargo.lock | 11 ++ world/Cargo.toml | 8 +- world/examples/pricing_csv.rs | 2 +- world/examples/world_block_statistics.rs | 173 +++++++++++++++++++++++ 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 world/examples/world_block_statistics.rs diff --git a/Cargo.lock b/Cargo.lock index 6b9630ecd3..65f5c8fe67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,6 +2658,15 @@ dependencies = [ "libloading 0.7.0", ] +[[package]] +name = "kiddo" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e70400c6435a3f5e4d7bfb406af50bc66950cc8f24f45488d66f47bf3f5e25" +dependencies = [ + "num-traits", +] + [[package]] name = "lazy-bytes-cast" version = "5.0.1" @@ -6138,11 +6147,13 @@ dependencies = [ "csv", "deflate 0.9.1", "enum-iterator", + "fallible-iterator", "flate2", "fxhash", "hashbrown 0.11.2", "image", "itertools 0.10.0", + "kiddo", "lazy_static", "lz-fear", "minifb", diff --git a/world/Cargo.toml b/world/Cargo.toml index db64fc3724..56ff2134b6 100644 --- a/world/Cargo.toml +++ b/world/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [features] simd = ["vek/platform_intrinsics"] -bin_compression = ["lz-fear", "deflate", "flate2", "image/jpeg", "num-traits"] +bin_compression = ["lz-fear", "deflate", "flate2", "image/jpeg", "num-traits", "fallible-iterator", "kiddo"] default = ["simd"] @@ -41,6 +41,8 @@ lz-fear = { version = "0.1.1", optional = true } deflate = { version = "0.9.1", optional = true } flate2 = { version = "1.0.20", optional = true } num-traits = { version = "0.2", optional = true } +fallible-iterator = { version = "0.2.0", optional = true } +kiddo = { version = "0.1.4", optional = true } [dev-dependencies] @@ -62,6 +64,10 @@ name = "tree" name = "chunk_compression_benchmarks" required-features = ["bin_compression"] +[[example]] +name = "world_block_statistics" +required-features = ["bin_compression"] + [[example]] name = "heightmap_visualization" required-features = ["bin_compression"] diff --git a/world/examples/pricing_csv.rs b/world/examples/pricing_csv.rs index c3e0195dd0..0565e4a99e 100644 --- a/world/examples/pricing_csv.rs +++ b/world/examples/pricing_csv.rs @@ -73,7 +73,7 @@ fn economy_sqlite(world: &World, index: &Index) -> Result<(), Box> { DROP TABLE IF EXISTS site; CREATE TABLE site ( xcoord INTEGER NOT NULL, - ycoord INTEGER NUT NULL, + ycoord INTEGER NOT NULL, name TEXT NOT NULL ); CREATE UNIQUE INDEX site_position ON site(xcoord, ycoord); diff --git a/world/examples/world_block_statistics.rs b/world/examples/world_block_statistics.rs new file mode 100644 index 0000000000..af5fed39db --- /dev/null +++ b/world/examples/world_block_statistics.rs @@ -0,0 +1,173 @@ +use common::{ + terrain::TerrainChunkSize, + vol::{IntoVolIterator, RectVolSize}, +}; +use fallible_iterator::FallibleIterator; +use kiddo::{distance::squared_euclidean, KdTree}; +use rayon::{ + iter::{IntoParallelIterator, ParallelIterator}, + ThreadPoolBuilder, +}; +use rusqlite::{Connection, ToSql, NO_PARAMS}; +use std::{ + collections::{HashMap, HashSet}, + error::Error, + sync::mpsc, + time::{SystemTime, UNIX_EPOCH}, +}; +use vek::*; +use veloren_world::{ + sim::{FileOpts, WorldOpts, DEFAULT_WORLD_MAP}, + World, +}; + +fn block_statistics_db() -> Result> { + let conn = Connection::open("block_statistics.sqlite")?; + #[rustfmt::skip] + conn.execute_batch(" + CREATE TABLE IF NOT EXISTS chunk ( + xcoord INTEGER NOT NULL, + ycoord INTEGER NOT NULL, + height INTEGER NOT NULL, + start_time REAL NOT NULL, + end_time REAL NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS chunk_position ON chunk(xcoord, ycoord); + CREATE TABLE IF NOT EXISTS block ( + xcoord INTEGER NOT NULL, + ycoord INTEGER NOT NULL, + kind TEXT NOT NULL, + r INTEGER NOT NULL, + g INTEGER NOT NULL, + b INTEGER NOT NULL, + quantity INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS block_position ON block(xcoord, ycoord, kind, r, g, b); + CREATE TABLE IF NOT EXISTS sprite ( + xcoord INTEGER NOT NULL, + ycoord INTEGER NOT NULL, + kind TEXT NOT NULL, + quantity INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS sprite_position ON sprite(xcoord, ycoord, kind); + ")?; + Ok(conn) +} + +fn main() -> Result<(), Box> { + common_frontend::init_stdout(None); + println!("Loading world"); + let pool = ThreadPoolBuilder::new().build().unwrap(); + let (world, index) = World::generate( + 59686, + WorldOpts { + seed_elements: true, + world_file: FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into()), + }, + &pool, + ); + println!("Loaded world"); + + let conn = block_statistics_db()?; + + let existing_chunks: HashSet<(i32, i32)> = conn + .prepare("SELECT xcoord, ycoord FROM chunk")? + .query(NO_PARAMS)? + .map(|row| Ok((row.get(0)?, row.get(1)?))) + .collect()?; + + let sz = world.sim().get_size(); + let (tx, rx) = mpsc::channel(); + rayon::spawn(move || { + let coords: Vec<_> = (1..sz.y) + .into_iter() + .flat_map(move |y| { + let tx = tx.clone(); + (1..sz.x) + .into_iter() + .map(move |x| (tx.clone(), x as i32, y as i32)) + }) + .collect(); + coords.into_par_iter().for_each(|(tx, x, y)| { + if existing_chunks.contains(&(x, y)) { + return; + } + println!("Generating chunk at ({}, {})", x, y); + let start_time = SystemTime::now(); + if let Ok((chunk, _supplement)) = + world.generate_chunk(index.as_index_ref(), Vec2::new(x, y), || false, None) + { + let end_time = SystemTime::now(); + // TODO: can kiddo be made to work without the `Float` bound, so we can use + // `KdTree` (currently it uses 15 bytes per point instead of 3)? + let mut block_colors = KdTree::, 3>::new(); + let mut block_counts = HashMap::new(); + let mut sprite_counts = HashMap::new(); + let lo = Vec3::new(0, 0, chunk.get_min_z()); + let hi = TerrainChunkSize::RECT_SIZE.as_().with_z(chunk.get_max_z()); + let height = chunk.get_max_z() - chunk.get_min_z(); + for (_, block) in chunk.vol_iter(lo, hi) { + let mut rgb = block.get_color().unwrap_or(Rgb::new(0, 0, 0)); + let color: [f32; 3] = [rgb.r as _, rgb.g as _, rgb.b as _]; + if let Ok((dist, nearest)) = + block_colors.nearest_one(&color, &squared_euclidean) + { + if dist < (5.0f32).powf(2.0) { + rgb = *nearest; + } + } + let _ = block_colors.add(&color, rgb); + *block_counts.entry((block.kind(), rgb)).or_insert(0) += 1; + if let Some(sprite) = block.get_sprite() { + *sprite_counts.entry(sprite).or_insert(0) += 1; + } + } + let _ = tx.send(( + x, + y, + height, + start_time, + end_time, + block_counts, + sprite_counts, + )); + } + }); + }); + #[rustfmt::skip] + let mut insert_block = conn.prepare(" + REPLACE INTO block (xcoord, ycoord, kind, r, g, b, quantity) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ")?; + #[rustfmt::skip] + let mut insert_sprite = conn.prepare(" + REPLACE INTO sprite (xcoord, ycoord, kind, quantity) + VALUES (?1, ?2, ?3, ?4) + ")?; + #[rustfmt::skip] + let mut insert_chunk = conn.prepare(" + REPLACE INTO chunk (xcoord, ycoord, height, start_time, end_time) + VALUES (?1, ?2, ?3, ?4, ?5) + ")?; + while let Ok((x, y, height, start_time, end_time, block_counts, sprite_counts)) = rx.recv() { + println!("Inserting results for chunk at ({}, {})", x, y); + for ((kind, color), count) in block_counts.iter() { + insert_block.execute(&[ + &x as &dyn ToSql, + &y, + &format!("{:?}", kind), + &color.r, + &color.g, + &color.b, + &count, + ])?; + } + for (kind, count) in sprite_counts.iter() { + insert_sprite.execute(&[&x as &dyn ToSql, &y, &format!("{:?}", kind), &count])?; + } + let start_time = start_time.duration_since(UNIX_EPOCH)?.as_secs_f64(); + let end_time = end_time.duration_since(UNIX_EPOCH)?.as_secs_f64(); + insert_chunk.execute(&[&x as &dyn ToSql, &y, &height, &start_time, &end_time])?; + } + Ok(()) +}