Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

rayon::join creates a global threadpool, which is only used in /world

Browse Source 
instead just use the same threadpool for everything
helps with debugging problems with GDB
using threadpool.install() to also be used when `into_par_iter()` is called
This commit is contained in:
Marcel Märtens 2021-05-04 17:11:01 +02:00
parent 101fb498a5
commit 9fefdcbbca
11 changed files with 203 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -5821,6 +5821,7 @@ dependencies = [
"old_school_gfx_glutin_ext", "old_school_gfx_glutin_ext",
"ordered-float 2.1.1", "ordered-float 2.1.1",
"rand 0.8.3", "rand 0.8.3",
"rayon",
"rodio", "rodio",
"ron", "ron",
"serde", "serde",

2
common/state/src/state.rs
View File

@ -321,6 +321,8 @@ impl State {
/// Get a mutable reference to the internal ECS world. /// Get a mutable reference to the internal ECS world.
pub fn ecs_mut(&mut self) -> &mut specs::World { &mut self.ecs } pub fn ecs_mut(&mut self) -> &mut specs::World { &mut self.ecs }
pub fn thread_pool(&self) -> Arc<ThreadPool> { Arc::clone(&self.thread_pool) }
/// Get a reference to the `TerrainChanges` structure of the state. This /// Get a reference to the `TerrainChanges` structure of the state. This
/// contains information about terrain state that has changed since the /// contains information about terrain state that has changed since the
/// last game tick. /// last game tick.

27
server/src/lib.rs
View File

@ -282,21 +282,26 @@ impl Server {
state.ecs_mut().insert(AliasValidator::new(banned_words)); state.ecs_mut().insert(AliasValidator::new(banned_words));
#[cfg(feature = "worldgen")] #[cfg(feature = "worldgen")]
let (world, index) = World::generate(settings.world_seed, WorldOpts { let (world, index) = World::generate(
seed_elements: true, settings.world_seed,
world_file: if let Some(ref opts) = settings.map_file { WorldOpts {
opts.clone() seed_elements: true,
} else { world_file: if let Some(ref opts) = settings.map_file {
// Load default map from assets. opts.clone()
FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into()) } else {
// Load default map from assets.
FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into())
},
..WorldOpts::default()
}, },
..WorldOpts::default() &state.thread_pool(),
}); );
#[cfg(feature = "worldgen")] #[cfg(feature = "worldgen")]
let map = world.get_map_data(index.as_index_ref()); let map = world.get_map_data(index.as_index_ref(), &state.thread_pool());
#[cfg(not(feature = "worldgen"))] #[cfg(not(feature = "worldgen"))]
let (world, index) = World::generate(settings.world_seed); let (world, index) = World::generate(settings.world_seed, &state.thread_pool());
#[cfg(not(feature = "worldgen"))] #[cfg(not(feature = "worldgen"))]
let map = WorldMapMsg { let map = WorldMapMsg {
dimensions_lg: Vec2::zero(), dimensions_lg: Vec2::zero(),

1
voxygen/Cargo.toml
View File

@ -123,6 +123,7 @@ winres = "0.1"
criterion = "0.3" criterion = "0.3"
git2 = "0.13" git2 = "0.13"
world = {package = "veloren-world", path = "../world"} world = {package = "veloren-world", path = "../world"}
rayon = "1.5.0"
[[bench]] [[bench]]
harness = false harness = false

21
voxygen/benches/meshing_benchmark.rs
View File

@ -13,16 +13,21 @@ const GEN_SIZE: i32 = 4;
#[allow(clippy::needless_update)] // TODO: Pending review in #587 #[allow(clippy::needless_update)] // TODO: Pending review in #587
pub fn criterion_benchmark(c: &mut Criterion) { pub fn criterion_benchmark(c: &mut Criterion) {
let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
// Generate chunks here to test // Generate chunks here to test
let mut terrain = TerrainGrid::new().unwrap(); let mut terrain = TerrainGrid::new().unwrap();
let (world, index) = World::generate(42, sim::WorldOpts { let (world, index) = World::generate(
// NOTE: If this gets too expensive, we can turn it off. 42,
// TODO: Consider an option to turn off all erosion as well, or even provide altitude sim::WorldOpts {
// directly with a closure. // NOTE: If this gets too expensive, we can turn it off.
seed_elements: true, // TODO: Consider an option to turn off all erosion as well, or even provide altitude
world_file: sim::FileOpts::LoadAsset(sim::DEFAULT_WORLD_MAP.into()), // directly with a closure.
..Default::default() seed_elements: true,
}); world_file: sim::FileOpts::LoadAsset(sim::DEFAULT_WORLD_MAP.into()),
..Default::default()
},
&pool,
);
let index = index.as_index_ref(); let index = index.as_index_ref();
(0..GEN_SIZE) (0..GEN_SIZE)
.flat_map(|x| (0..GEN_SIZE).map(move |y| Vec2::new(x, y))) .flat_map(|x| (0..GEN_SIZE).map(move |y| Vec2::new(x, y)))

13
world/examples/view.rs
View File

@ -6,10 +6,15 @@ const W: usize = 640;
const H: usize = 480; const H: usize = 480;
fn main() { fn main() {
let (world, index) = World::generate(0, WorldOpts { let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
seed_elements: true, let (world, index) = World::generate(
..WorldOpts::default() 0,
}); WorldOpts {
seed_elements: true,
..WorldOpts::default()
},
&threadpool,
);
let index = index.as_index_ref(); let index = index.as_index_ref();

19
world/examples/water.rs
View File

@ -29,6 +29,7 @@ fn main() {
.with_max_level(Level::ERROR) .with_max_level(Level::ERROR)
.with_env_filter(EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into())) .with_env_filter(EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into()))
.init(); .init();
let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
// To load a map file of your choice, replace map_file with the name of your map // To load a map file of your choice, replace map_file with the name of your map
// (stored locally in the map directory of your Veloren root), and swap the // (stored locally in the map directory of your Veloren root), and swap the
@ -42,13 +43,17 @@ fn main() {
let mut _map_file = PathBuf::from("./maps"); let mut _map_file = PathBuf::from("./maps");
_map_file.push(map_file); _map_file.push(map_file);
let (world, index) = World::generate(5284, WorldOpts { let (world, index) = World::generate(
seed_elements: false, 5284,
world_file: sim::FileOpts::LoadAsset(veloren_world::sim::DEFAULT_WORLD_MAP.into()), WorldOpts {
// world_file: sim::FileOpts::Load(_map_file), seed_elements: false,
// world_file: sim::FileOpts::Save, world_file: sim::FileOpts::LoadAsset(veloren_world::sim::DEFAULT_WORLD_MAP.into()),
..WorldOpts::default() // world_file: sim::FileOpts::Load(_map_file),
}); // world_file: sim::FileOpts::Save,
..WorldOpts::default()
},
&threadpool,
);
let index = index.as_index_ref(); let index = index.as_index_ref();
tracing::info!("Sampling data..."); tracing::info!("Sampling data...");
let sampler = world.sim(); let sampler = world.sim();

138
world/src/lib.rs
View File

@ -87,16 +87,24 @@ impl assets::Asset for Colors {
} }
impl World { impl World {
pub fn generate(seed: u32, opts: sim::WorldOpts) -> (Self, IndexOwned) { pub fn generate(
seed: u32,
opts: sim::WorldOpts,
threadpool: &rayon::ThreadPool,
) -> (Self, IndexOwned) {
// NOTE: Generating index first in order to quickly fail if the color manifest // NOTE: Generating index first in order to quickly fail if the color manifest
// is broken. // is broken.
let mut index = Index::new(seed); threadpool.install(|| {
let mut sim = sim::WorldSim::generate(seed, opts); let mut index = Index::new(seed);
let civs = civ::Civs::generate(seed, &mut sim, &mut index);
sim2::simulate(&mut index, &mut sim); let mut sim = sim::WorldSim::generate(seed, opts, threadpool);
(Self { sim, civs }, IndexOwned::new(index)) let civs = civ::Civs::generate(seed, &mut sim, &mut index);
sim2::simulate(&mut index, &mut sim);
(Self { sim, civs }, IndexOwned::new(index))
})
} }
pub fn sim(&self) -> &sim::WorldSim { &self.sim } pub fn sim(&self) -> &sim::WorldSim { &self.sim }
@ -107,66 +115,68 @@ impl World {
// TODO // TODO
} }
pub fn get_map_data(&self, index: IndexRef) -> WorldMapMsg { pub fn get_map_data(&self, index: IndexRef, threadpool: &rayon::ThreadPool) -> WorldMapMsg {
// we need these numbers to create unique ids for cave ends threadpool.install(|| {
let num_sites = self.civs().sites().count() as u64; // we need these numbers to create unique ids for cave ends
let num_caves = self.civs().caves.values().count() as u64; let num_sites = self.civs().sites().count() as u64;
WorldMapMsg { let num_caves = self.civs().caves.values().count() as u64;
pois: self.civs().pois.iter().map(|(_, poi)| { WorldMapMsg {
world_msg::PoiInfo { pois: self.civs().pois.iter().map(|(_, poi)| {
name: poi.name.clone(), world_msg::PoiInfo {
kind: match &poi.kind { name: poi.name.clone(),
civ::PoiKind::Peak(alt) => world_msg::PoiKind::Peak(*alt), kind: match &poi.kind {
civ::PoiKind::Lake(size) => world_msg::PoiKind::Lake(*size), civ::PoiKind::Peak(alt) => world_msg::PoiKind::Peak(*alt),
}, civ::PoiKind::Lake(size) => world_msg::PoiKind::Lake(*size),
wpos: poi.loc * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
}
}).collect(),
sites: self
.civs()
.sites
.iter()
.map(|(_, site)| {
world_msg::SiteInfo {
id: site.site_tmp.map(|i| i.id()).unwrap_or_default(),
name: site.site_tmp.map(|id| index.sites[id].name().to_string()),
// TODO: Probably unify these, at some point
kind: match &site.kind {
civ::SiteKind::Settlement => world_msg::SiteKind::Town,
civ::SiteKind::Dungeon => world_msg::SiteKind::Dungeon {
difficulty: match site.site_tmp.map(|id| &index.sites[id].kind) {
Some(site::SiteKind::Dungeon(d)) => d.difficulty(),
_ => 0,
},
},
civ::SiteKind::Castle => world_msg::SiteKind::Castle,
civ::SiteKind::Refactor => world_msg::SiteKind::Town,
civ::SiteKind::Tree => world_msg::SiteKind::Tree,
}, },
wpos: site.center * TerrainChunkSize::RECT_SIZE.map(|e| e as i32), wpos: poi.loc * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
} }
}) }).collect(),
.chain( sites: self
self.civs() .civs()
.caves .sites
.iter() .iter()
.map(|(id, info)| { .map(|(_, site)| {
// separate the two locations, combine with name world_msg::SiteInfo {
std::iter::once((id.id()+num_sites, info.name.clone(), info.location.0)) id: site.site_tmp.map(|i| i.id()).unwrap_or_default(),
// unfortunately we have to introduce a fake id (as it gets stored in a map in the client) name: site.site_tmp.map(|id| index.sites[id].name().to_string()),
.chain(std::iter::once((id.id()+num_sites+num_caves, info.name.clone(), info.location.1))) // TODO: Probably unify these, at some point
}) kind: match &site.kind {
.flatten() // unwrap inner iteration civ::SiteKind::Settlement => world_msg::SiteKind::Town,
.map(|(id, name, pos)| world_msg::SiteInfo { civ::SiteKind::Dungeon => world_msg::SiteKind::Dungeon {
id, difficulty: match site.site_tmp.map(|id| &index.sites[id].kind) {
name: Some(name), Some(site::SiteKind::Dungeon(d)) => d.difficulty(),
kind: world_msg::SiteKind::Cave, _ => 0,
wpos: pos, },
}), },
) civ::SiteKind::Castle => world_msg::SiteKind::Castle,
.collect(), civ::SiteKind::Refactor => world_msg::SiteKind::Town,
..self.sim.get_map(index) civ::SiteKind::Tree => world_msg::SiteKind::Tree,
} },
wpos: site.center * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
}
})
.chain(
self.civs()
.caves
.iter()
.map(|(id, info)| {
// separate the two locations, combine with name
std::iter::once((id.id() + num_sites, info.name.clone(), info.location.0))
// unfortunately we have to introduce a fake id (as it gets stored in a map in the client)
.chain(std::iter::once((id.id() + num_sites + num_caves, info.name.clone(), info.location.1)))
})
.flatten() // unwrap inner iteration
.map(|(id, name, pos)| world_msg::SiteInfo {
id,
name: Some(name),
kind: world_msg::SiteKind::Cave,
wpos: pos,
}),
)
.collect(),
..self.sim.get_map(index)
}
})
} }
pub fn sample_columns( pub fn sample_columns(

15
world/src/sim/erosion.rs
View File

@ -710,6 +710,7 @@ fn erode(
// scaling factors // scaling factors
height_scale: impl Fn(f32) -> Alt + Sync, height_scale: impl Fn(f32) -> Alt + Sync,
k_da_scale: impl Fn(f64) -> f64, k_da_scale: impl Fn(f64) -> f64,
threadpool: &rayon::ThreadPool,
) { ) {
let compute_stats = true; let compute_stats = true;
debug!("Done draining..."); debug!("Done draining...");
@ -795,7 +796,7 @@ fn erode(
let k_fs_mult_sed = 4.0; let k_fs_mult_sed = 4.0;
// Dimensionless multiplier for G when land becomes sediment. // Dimensionless multiplier for G when land becomes sediment.
let g_fs_mult_sed = 1.0; let g_fs_mult_sed = 1.0;
let ((dh, newh, maxh, mrec, mstack, mwrec, area), (mut max_slopes, h_t)) = rayon::join( let ((dh, newh, maxh, mrec, mstack, mwrec, area), (mut max_slopes, h_t)) = threadpool.join(
|| { || {
let mut dh = downhill( let mut dh = downhill(
map_size_lg, map_size_lg,
@ -817,6 +818,7 @@ fn erode(
dx as Compute, dx as Compute,
dy as Compute, dy as Compute,
maxh, maxh,
threadpool,
); );
debug!("Got multiple receivers..."); debug!("Got multiple receivers...");
// TODO: Figure out how to switch between single-receiver and multi-receiver // TODO: Figure out how to switch between single-receiver and multi-receiver
@ -827,7 +829,7 @@ fn erode(
(dh, newh, maxh, mrec, mstack, mwrec, area) (dh, newh, maxh, mrec, mstack, mwrec, area)
}, },
|| { || {
rayon::join( threadpool.join(
|| { || {
let max_slope = get_max_slope(map_size_lg, h, rock_strength_nz, |posi| { let max_slope = get_max_slope(map_size_lg, h, rock_strength_nz, |posi| {
height_scale(n_f(posi)) height_scale(n_f(posi))
@ -851,7 +853,7 @@ fn erode(
// Precompute factors for Stream Power Law. // Precompute factors for Stream Power Law.
let czero = <SimdType as Zero>::zero(); let czero = <SimdType as Zero>::zero();
let (k_fs_fact, k_df_fact) = rayon::join( let (k_fs_fact, k_df_fact) = threadpool.join(
|| { || {
dh.par_iter() dh.par_iter()
.enumerate() .enumerate()
@ -1081,7 +1083,7 @@ fn erode(
// Keep track of how many iterations we've gone to test for convergence. // Keep track of how many iterations we've gone to test for convergence.
n_gs_stream_power_law += 1; n_gs_stream_power_law += 1;
rayon::join( threadpool.join(
|| { || {
// guess/update the elevation at t+Δt (k) // guess/update the elevation at t+Δt (k)
(&mut *h_p, &*h_stack) (&mut *h_p, &*h_stack)
@ -2339,6 +2341,7 @@ pub fn get_multi_rec<F: fmt::Debug + Float + Sync + Into<Compute>>(
dx: Compute, dx: Compute,
dy: Compute, dy: Compute,
_maxh: F, _maxh: F,
threadpool: &rayon::ThreadPool,
) -> (Box<[u8]>, Box<[u32]>, Box<[Computex8]>) { ) -> (Box<[u8]>, Box<[u32]>, Box<[Computex8]>) {
let nn = nx * ny; let nn = nx * ny;
let dxdy = Vec2::new(dx, dy); let dxdy = Vec2::new(dx, dy);
@ -2437,7 +2440,7 @@ pub fn get_multi_rec<F: fmt::Debug + Float + Sync + Into<Compute>>(
.unzip_into_vecs(&mut mrec, &mut don_vis); .unzip_into_vecs(&mut mrec, &mut don_vis);
let czero = <Compute as Zero>::zero(); let czero = <Compute as Zero>::zero();
let (wrec, stack) = rayon::join( let (wrec, stack) = threadpool.join(
|| { || {
(0..nn) (0..nn)
.into_par_iter() .into_par_iter()
@ -2541,6 +2544,7 @@ pub fn do_erosion(
height_scale: impl Fn(f32) -> Alt + Sync, height_scale: impl Fn(f32) -> Alt + Sync,
k_d_scale: f64, k_d_scale: f64,
k_da_scale: impl Fn(f64) -> f64, k_da_scale: impl Fn(f64) -> f64,
threadpool: &rayon::ThreadPool,
) -> (Box<[Alt]>, Box<[Alt]> /* , Box<[Alt]> */) { ) -> (Box<[Alt]>, Box<[Alt]> /* , Box<[Alt]> */) {
debug!("Initializing erosion arrays..."); debug!("Initializing erosion arrays...");
let oldh_ = (0..map_size_lg.chunks_len()) let oldh_ = (0..map_size_lg.chunks_len())
@ -2663,6 +2667,7 @@ pub fn do_erosion(
|posi| is_ocean(posi), |posi| is_ocean(posi),
height_scale, height_scale,
k_da_scale, k_da_scale,
threadpool,
); );
}); });
(h, b) (h, b)

120
world/src/sim/mod.rs
View File

@ -368,7 +368,7 @@ pub struct WorldSim {
impl WorldSim { impl WorldSim {
#[allow(clippy::unnested_or_patterns)] // TODO: Pending review in #587 #[allow(clippy::unnested_or_patterns)] // TODO: Pending review in #587
pub fn generate(seed: u32, opts: WorldOpts) -> Self { pub fn generate(seed: u32, opts: WorldOpts, threadpool: &rayon::ThreadPool) -> Self {
// Parse out the contents of various map formats into the values we need. // Parse out the contents of various map formats into the values we need.
let parsed_world_file = (|| { let parsed_world_file = (|| {
let map = match opts.world_file { let map = match opts.world_file {
@ -619,7 +619,7 @@ impl WorldSim {
// No NaNs in these uniform vectors, since the original noise value always // No NaNs in these uniform vectors, since the original noise value always
// returns Some. // returns Some.
let ((alt_base, _), (chaos, _)) = rayon::join( let ((alt_base, _), (chaos, _)) = threadpool.join(
|| { || {
uniform_noise(map_size_lg, |_, wposf| { uniform_noise(map_size_lg, |_, wposf| {
// "Base" of the chunk, to be multiplied by CONFIG.mountain_scale (multiplied // "Base" of the chunk, to be multiplied by CONFIG.mountain_scale (multiplied
@ -1067,6 +1067,7 @@ impl WorldSim {
height_scale, height_scale,
k_d_scale(n_approx), k_d_scale(n_approx),
k_da_scale, k_da_scale,
threadpool,
); );
// Quick "small scale" erosion cycle in order to lower extreme angles. // Quick "small scale" erosion cycle in order to lower extreme angles.
@ -1090,6 +1091,7 @@ impl WorldSim {
height_scale, height_scale,
k_d_scale(n_approx), k_d_scale(n_approx),
k_da_scale, k_da_scale,
threadpool,
) )
}; };
@ -1168,6 +1170,7 @@ impl WorldSim {
height_scale, height_scale,
k_d_scale(n_approx), k_d_scale(n_approx),
k_da_scale, k_da_scale,
threadpool,
) )
}; };
@ -1190,6 +1193,7 @@ impl WorldSim {
TerrainChunkSize::RECT_SIZE.x as Compute, TerrainChunkSize::RECT_SIZE.x as Compute,
TerrainChunkSize::RECT_SIZE.y as Compute, TerrainChunkSize::RECT_SIZE.y as Compute,
maxh, maxh,
threadpool,
) )
}; };
let flux_old = get_multi_drainage(map_size_lg, &mstack, &mrec, &*mwrec, boundary_len); let flux_old = get_multi_drainage(map_size_lg, &mstack, &mrec, &*mwrec, boundary_len);
@ -1318,61 +1322,63 @@ impl WorldSim {
}; };
// NaNs in these uniform vectors wherever pure_water() returns true. // NaNs in these uniform vectors wherever pure_water() returns true.
let (((alt_no_water, _), (pure_flux, _)), ((temp_base, _), (humid_base, _))) = rayon::join( let (((alt_no_water, _), (pure_flux, _)), ((temp_base, _), (humid_base, _))) = threadpool
|| { .join(
rayon::join( || {
|| { threadpool.join(
uniform_noise(map_size_lg, |posi, _| { || {
if pure_water(posi) { uniform_noise(map_size_lg, |posi, _| {
None if pure_water(posi) {
} else { None
// A version of alt that is uniform over *non-water* (or } else {
// land-adjacent water) chunks. // A version of alt that is uniform over *non-water* (or
Some(alt[posi] as f32) // land-adjacent water) chunks.
} Some(alt[posi] as f32)
}) }
}, })
|| { },
uniform_noise(map_size_lg, |posi, _| { || {
if pure_water(posi) { uniform_noise(map_size_lg, |posi, _| {
None if pure_water(posi) {
} else { None
Some(flux_old[posi]) } else {
} Some(flux_old[posi])
}) }
}, })
) },
}, )
|| { },
rayon::join( || {
|| { threadpool.join(
uniform_noise(map_size_lg, |posi, wposf| { || {
if pure_water(posi) { uniform_noise(map_size_lg, |posi, wposf| {
None if pure_water(posi) {
} else { None
// -1 to 1. } else {
Some(gen_ctx.temp_nz.get((wposf).into_array()) as f32) // -1 to 1.
} Some(gen_ctx.temp_nz.get((wposf).into_array()) as f32)
}) }
}, })
|| { },
uniform_noise(map_size_lg, |posi, wposf| { || {
// Check whether any tiles around this tile are water. uniform_noise(map_size_lg, |posi, wposf| {
if pure_water(posi) { // Check whether any tiles around this tile are water.
None if pure_water(posi) {
} else { None
// 0 to 1, hopefully. } else {
Some( // 0 to 1, hopefully.
(gen_ctx.humid_nz.get(wposf.div(1024.0).into_array()) as f32) Some(
.add(1.0) (gen_ctx.humid_nz.get(wposf.div(1024.0).into_array())
.mul(0.5), as f32)
) .add(1.0)
} .mul(0.5),
}) )
}, }
) })
}, },
); )
},
);
let gen_cdf = GenCdf { let gen_cdf = GenCdf {
humid_base, humid_base,

3
world/src/sim2/mod.rs
View File

@ -1071,6 +1071,7 @@ mod tests {
#[test] #[test]
fn test_economy() { fn test_economy() {
init(); init();
let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
info!("init"); info!("init");
let seed = 59686; let seed = 59686;
let opts = sim::WorldOpts { let opts = sim::WorldOpts {
@ -1080,7 +1081,7 @@ mod tests {
}; };
let mut index = crate::index::Index::new(seed); let mut index = crate::index::Index::new(seed);
info!("Index created"); info!("Index created");
let mut sim = sim::WorldSim::generate(seed, opts); let mut sim = sim::WorldSim::generate(seed, opts, &threadpool);
info!("World loaded"); info!("World loaded");
let regenerate_input = false; let regenerate_input = false;
if regenerate_input { if regenerate_input {