Mirrored_Repos/veloren
1
0
Fork 0
mirror of https://gitlab.com/veloren/veloren.git synced 2024-08-30 18:12:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

rayon::join creates a global threadpool, which is only used in /world

Browse Source 
instead just use the same threadpool for everything
helps with debugging problems with GDB
using threadpool.install() to also be used when `into_par_iter()` is called
This commit is contained in:
Marcel Märtens 2021-05-04 17:11:01 +02:00
parent fb940ad27a
commit 6e3a74b476
11 changed files with 203 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -5821,6 +5821,7 @@ dependencies = [
"old_school_gfx_glutin_ext",
"ordered-float 2.1.1",
"rand 0.8.3",
"rayon",
"rodio",
"ron",
"serde",

2
common/state/src/state.rs
View File

@ -321,6 +321,8 @@ impl State {
/// Get a mutable reference to the internal ECS world.
pub fn ecs_mut(&mut self) -> &mut specs::World { &mut self.ecs }
pub fn thread_pool(&self) -> Arc<ThreadPool> { Arc::clone(&self.thread_pool) }
/// Get a reference to the `TerrainChanges` structure of the state. This
/// contains information about terrain state that has changed since the
/// last game tick.

27
server/src/lib.rs
View File

@ -282,21 +282,26 @@ impl Server {
state.ecs_mut().insert(AliasValidator::new(banned_words));
#[cfg(feature = "worldgen")]
let (world, index) = World::generate(settings.world_seed, WorldOpts {
seed_elements: true,
world_file: if let Some(ref opts) = settings.map_file {
opts.clone()
} else {
// Load default map from assets.
FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into())
let (world, index) = World::generate(
settings.world_seed,
WorldOpts {
seed_elements: true,
world_file: if let Some(ref opts) = settings.map_file {
opts.clone()
} else {
// Load default map from assets.
FileOpts::LoadAsset(DEFAULT_WORLD_MAP.into())
},
..WorldOpts::default()
},
..WorldOpts::default()
});
&state.thread_pool(),
);
#[cfg(feature = "worldgen")]
let map = world.get_map_data(index.as_index_ref());
let map = world.get_map_data(index.as_index_ref(), &state.thread_pool());
#[cfg(not(feature = "worldgen"))]
let (world, index) = World::generate(settings.world_seed);
let (world, index) = World::generate(settings.world_seed, &state.thread_pool());
#[cfg(not(feature = "worldgen"))]
let map = WorldMapMsg {
dimensions_lg: Vec2::zero(),

1
voxygen/Cargo.toml
View File

@ -123,6 +123,7 @@ winres = "0.1"
criterion = "0.3"
git2 = "0.13"
world = {package = "veloren-world", path = "../world"}
rayon = "1.5.0"
[[bench]]
harness = false

21
voxygen/benches/meshing_benchmark.rs
View File

@ -13,16 +13,21 @@ const GEN_SIZE: i32 = 4;
#[allow(clippy::needless_update)] // TODO: Pending review in #587
pub fn criterion_benchmark(c: &mut Criterion) {
let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
// Generate chunks here to test
let mut terrain = TerrainGrid::new().unwrap();
let (world, index) = World::generate(42, sim::WorldOpts {
// NOTE: If this gets too expensive, we can turn it off.
// TODO: Consider an option to turn off all erosion as well, or even provide altitude
// directly with a closure.
seed_elements: true,
world_file: sim::FileOpts::LoadAsset(sim::DEFAULT_WORLD_MAP.into()),
..Default::default()
});
let (world, index) = World::generate(
42,
sim::WorldOpts {
// NOTE: If this gets too expensive, we can turn it off.
// TODO: Consider an option to turn off all erosion as well, or even provide altitude
// directly with a closure.
seed_elements: true,
world_file: sim::FileOpts::LoadAsset(sim::DEFAULT_WORLD_MAP.into()),
..Default::default()
},
&pool,
);
let index = index.as_index_ref();
(0..GEN_SIZE)
.flat_map(|x| (0..GEN_SIZE).map(move |y| Vec2::new(x, y)))

13
world/examples/view.rs
View File

@ -6,10 +6,15 @@ const W: usize = 640;
const H: usize = 480;
fn main() {
let (world, index) = World::generate(0, WorldOpts {
seed_elements: true,
..WorldOpts::default()
});
let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
let (world, index) = World::generate(
0,
WorldOpts {
seed_elements: true,
..WorldOpts::default()
},
&threadpool,
);
let index = index.as_index_ref();

19
world/examples/water.rs
View File

@ -29,6 +29,7 @@ fn main() {
.with_max_level(Level::ERROR)
.with_env_filter(EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into()))
.init();
let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
// To load a map file of your choice, replace map_file with the name of your map
// (stored locally in the map directory of your Veloren root), and swap the
@ -42,13 +43,17 @@ fn main() {
let mut _map_file = PathBuf::from("./maps");
_map_file.push(map_file);
let (world, index) = World::generate(5284, WorldOpts {
seed_elements: false,
world_file: sim::FileOpts::LoadAsset(veloren_world::sim::DEFAULT_WORLD_MAP.into()),
// world_file: sim::FileOpts::Load(_map_file),
// world_file: sim::FileOpts::Save,
..WorldOpts::default()
});
let (world, index) = World::generate(
5284,
WorldOpts {
seed_elements: false,
world_file: sim::FileOpts::LoadAsset(veloren_world::sim::DEFAULT_WORLD_MAP.into()),
// world_file: sim::FileOpts::Load(_map_file),
// world_file: sim::FileOpts::Save,
..WorldOpts::default()
},
&threadpool,
);
let index = index.as_index_ref();
tracing::info!("Sampling data...");
let sampler = world.sim();

138
world/src/lib.rs
View File

@ -87,16 +87,24 @@ impl assets::Asset for Colors {
}
impl World {
pub fn generate(seed: u32, opts: sim::WorldOpts) -> (Self, IndexOwned) {
pub fn generate(
seed: u32,
opts: sim::WorldOpts,
threadpool: &rayon::ThreadPool,
) -> (Self, IndexOwned) {
// NOTE: Generating index first in order to quickly fail if the color manifest
// is broken.
let mut index = Index::new(seed);
let mut sim = sim::WorldSim::generate(seed, opts);
let civs = civ::Civs::generate(seed, &mut sim, &mut index);
threadpool.install(|| {
let mut index = Index::new(seed);
sim2::simulate(&mut index, &mut sim);
let mut sim = sim::WorldSim::generate(seed, opts, threadpool);
(Self { sim, civs }, IndexOwned::new(index))
let civs = civ::Civs::generate(seed, &mut sim, &mut index);
sim2::simulate(&mut index, &mut sim);
(Self { sim, civs }, IndexOwned::new(index))
})
}
pub fn sim(&self) -> &sim::WorldSim { &self.sim }
@ -107,66 +115,68 @@ impl World {
// TODO
}
pub fn get_map_data(&self, index: IndexRef) -> WorldMapMsg {
// we need these numbers to create unique ids for cave ends
let num_sites = self.civs().sites().count() as u64;
let num_caves = self.civs().caves.values().count() as u64;
WorldMapMsg {
pois: self.civs().pois.iter().map(|(_, poi)| {
world_msg::PoiInfo {
name: poi.name.clone(),
kind: match &poi.kind {
civ::PoiKind::Peak(alt) => world_msg::PoiKind::Peak(*alt),
civ::PoiKind::Lake(size) => world_msg::PoiKind::Lake(*size),
},
wpos: poi.loc * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
}
}).collect(),
sites: self
.civs()
.sites
.iter()
.map(|(_, site)| {
world_msg::SiteInfo {
id: site.site_tmp.map(|i| i.id()).unwrap_or_default(),
name: site.site_tmp.map(|id| index.sites[id].name().to_string()),
// TODO: Probably unify these, at some point
kind: match &site.kind {
civ::SiteKind::Settlement => world_msg::SiteKind::Town,
civ::SiteKind::Dungeon => world_msg::SiteKind::Dungeon {
difficulty: match site.site_tmp.map(|id| &index.sites[id].kind) {
Some(site::SiteKind::Dungeon(d)) => d.difficulty(),
_ => 0,
},
},
civ::SiteKind::Castle => world_msg::SiteKind::Castle,
civ::SiteKind::Refactor => world_msg::SiteKind::Town,
civ::SiteKind::Tree => world_msg::SiteKind::Tree,
pub fn get_map_data(&self, index: IndexRef, threadpool: &rayon::ThreadPool) -> WorldMapMsg {
threadpool.install(|| {
// we need these numbers to create unique ids for cave ends
let num_sites = self.civs().sites().count() as u64;
let num_caves = self.civs().caves.values().count() as u64;
WorldMapMsg {
pois: self.civs().pois.iter().map(|(_, poi)| {
world_msg::PoiInfo {
name: poi.name.clone(),
kind: match &poi.kind {
civ::PoiKind::Peak(alt) => world_msg::PoiKind::Peak(*alt),
civ::PoiKind::Lake(size) => world_msg::PoiKind::Lake(*size),
},
wpos: site.center * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
wpos: poi.loc * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
}
})
.chain(
self.civs()
.caves
.iter()
.map(|(id, info)| {
// separate the two locations, combine with name
std::iter::once((id.id()+num_sites, info.name.clone(), info.location.0))
// unfortunately we have to introduce a fake id (as it gets stored in a map in the client)
.chain(std::iter::once((id.id()+num_sites+num_caves, info.name.clone(), info.location.1)))
})
.flatten() // unwrap inner iteration
.map(|(id, name, pos)| world_msg::SiteInfo {
id,
name: Some(name),
kind: world_msg::SiteKind::Cave,
wpos: pos,
}),
)
.collect(),
..self.sim.get_map(index)
}
}).collect(),
sites: self
.civs()
.sites
.iter()
.map(|(_, site)| {
world_msg::SiteInfo {
id: site.site_tmp.map(|i| i.id()).unwrap_or_default(),
name: site.site_tmp.map(|id| index.sites[id].name().to_string()),
// TODO: Probably unify these, at some point
kind: match &site.kind {
civ::SiteKind::Settlement => world_msg::SiteKind::Town,
civ::SiteKind::Dungeon => world_msg::SiteKind::Dungeon {
difficulty: match site.site_tmp.map(|id| &index.sites[id].kind) {
Some(site::SiteKind::Dungeon(d)) => d.difficulty(),
_ => 0,
},
},
civ::SiteKind::Castle => world_msg::SiteKind::Castle,
civ::SiteKind::Refactor => world_msg::SiteKind::Town,
civ::SiteKind::Tree => world_msg::SiteKind::Tree,
},
wpos: site.center * TerrainChunkSize::RECT_SIZE.map(|e| e as i32),
}
})
.chain(
self.civs()
.caves
.iter()
.map(|(id, info)| {
// separate the two locations, combine with name
std::iter::once((id.id() + num_sites, info.name.clone(), info.location.0))
// unfortunately we have to introduce a fake id (as it gets stored in a map in the client)
.chain(std::iter::once((id.id() + num_sites + num_caves, info.name.clone(), info.location.1)))
})
.flatten() // unwrap inner iteration
.map(|(id, name, pos)| world_msg::SiteInfo {
id,
name: Some(name),
kind: world_msg::SiteKind::Cave,
wpos: pos,
}),
)
.collect(),
..self.sim.get_map(index)
}
})
}
pub fn sample_columns(

15
world/src/sim/erosion.rs
View File

@ -710,6 +710,7 @@ fn erode(
// scaling factors
height_scale: impl Fn(f32) -> Alt + Sync,
k_da_scale: impl Fn(f64) -> f64,
threadpool: &rayon::ThreadPool,
) {
let compute_stats = true;
debug!("Done draining...");
@ -795,7 +796,7 @@ fn erode(
let k_fs_mult_sed = 4.0;
// Dimensionless multiplier for G when land becomes sediment.
let g_fs_mult_sed = 1.0;
let ((dh, newh, maxh, mrec, mstack, mwrec, area), (mut max_slopes, h_t)) = rayon::join(
let ((dh, newh, maxh, mrec, mstack, mwrec, area), (mut max_slopes, h_t)) = threadpool.join(
|| {
let mut dh = downhill(
map_size_lg,
@ -817,6 +818,7 @@ fn erode(
dx as Compute,
dy as Compute,
maxh,
threadpool,
);
debug!("Got multiple receivers...");
// TODO: Figure out how to switch between single-receiver and multi-receiver
@ -827,7 +829,7 @@ fn erode(
(dh, newh, maxh, mrec, mstack, mwrec, area)
},
|| {
rayon::join(
threadpool.join(
|| {
let max_slope = get_max_slope(map_size_lg, h, rock_strength_nz, |posi| {
height_scale(n_f(posi))
@ -851,7 +853,7 @@ fn erode(
// Precompute factors for Stream Power Law.
let czero = <SimdType as Zero>::zero();
let (k_fs_fact, k_df_fact) = rayon::join(
let (k_fs_fact, k_df_fact) = threadpool.join(
|| {
dh.par_iter()
.enumerate()
@ -1081,7 +1083,7 @@ fn erode(
// Keep track of how many iterations we've gone to test for convergence.
n_gs_stream_power_law += 1;
rayon::join(
threadpool.join(
|| {
// guess/update the elevation at t+Δt (k)
(&mut *h_p, &*h_stack)
@ -2339,6 +2341,7 @@ pub fn get_multi_rec<F: fmt::Debug + Float + Sync + Into<Compute>>(
dx: Compute,
dy: Compute,
_maxh: F,
threadpool: &rayon::ThreadPool,
) -> (Box<[u8]>, Box<[u32]>, Box<[Computex8]>) {
let nn = nx * ny;
let dxdy = Vec2::new(dx, dy);
@ -2437,7 +2440,7 @@ pub fn get_multi_rec<F: fmt::Debug + Float + Sync + Into<Compute>>(
.unzip_into_vecs(&mut mrec, &mut don_vis);
let czero = <Compute as Zero>::zero();
let (wrec, stack) = rayon::join(
let (wrec, stack) = threadpool.join(
|| {
(0..nn)
.into_par_iter()
@ -2541,6 +2544,7 @@ pub fn do_erosion(
height_scale: impl Fn(f32) -> Alt + Sync,
k_d_scale: f64,
k_da_scale: impl Fn(f64) -> f64,
threadpool: &rayon::ThreadPool,
) -> (Box<[Alt]>, Box<[Alt]> /* , Box<[Alt]> */) {
debug!("Initializing erosion arrays...");
let oldh_ = (0..map_size_lg.chunks_len())
@ -2663,6 +2667,7 @@ pub fn do_erosion(
|posi| is_ocean(posi),
height_scale,
k_da_scale,
threadpool,
);
});
(h, b)

120
world/src/sim/mod.rs
View File

@ -368,7 +368,7 @@ pub struct WorldSim {
impl WorldSim {
#[allow(clippy::unnested_or_patterns)] // TODO: Pending review in #587
pub fn generate(seed: u32, opts: WorldOpts) -> Self {
pub fn generate(seed: u32, opts: WorldOpts, threadpool: &rayon::ThreadPool) -> Self {
// Parse out the contents of various map formats into the values we need.
let parsed_world_file = (|| {
let map = match opts.world_file {
@ -619,7 +619,7 @@ impl WorldSim {
// No NaNs in these uniform vectors, since the original noise value always
// returns Some.
let ((alt_base, _), (chaos, _)) = rayon::join(
let ((alt_base, _), (chaos, _)) = threadpool.join(
|| {
uniform_noise(map_size_lg, |_, wposf| {
// "Base" of the chunk, to be multiplied by CONFIG.mountain_scale (multiplied
@ -1067,6 +1067,7 @@ impl WorldSim {
height_scale,
k_d_scale(n_approx),
k_da_scale,
threadpool,
);
// Quick "small scale" erosion cycle in order to lower extreme angles.
@ -1090,6 +1091,7 @@ impl WorldSim {
height_scale,
k_d_scale(n_approx),
k_da_scale,
threadpool,
)
};
@ -1168,6 +1170,7 @@ impl WorldSim {
height_scale,
k_d_scale(n_approx),
k_da_scale,
threadpool,
)
};
@ -1190,6 +1193,7 @@ impl WorldSim {
TerrainChunkSize::RECT_SIZE.x as Compute,
TerrainChunkSize::RECT_SIZE.y as Compute,
maxh,
threadpool,
)
};
let flux_old = get_multi_drainage(map_size_lg, &mstack, &mrec, &*mwrec, boundary_len);
@ -1318,61 +1322,63 @@ impl WorldSim {
};
// NaNs in these uniform vectors wherever pure_water() returns true.
let (((alt_no_water, _), (pure_flux, _)), ((temp_base, _), (humid_base, _))) = rayon::join(
|| {
rayon::join(
|| {
uniform_noise(map_size_lg, |posi, _| {
if pure_water(posi) {
None
} else {
// A version of alt that is uniform over *non-water* (or
// land-adjacent water) chunks.
Some(alt[posi] as f32)
}
})
},
|| {
uniform_noise(map_size_lg, |posi, _| {
if pure_water(posi) {
None
} else {
Some(flux_old[posi])
}
})
},
)
},
|| {
rayon::join(
|| {
uniform_noise(map_size_lg, |posi, wposf| {
if pure_water(posi) {
None
} else {
// -1 to 1.
Some(gen_ctx.temp_nz.get((wposf).into_array()) as f32)
}
})
},
|| {
uniform_noise(map_size_lg, |posi, wposf| {
// Check whether any tiles around this tile are water.
if pure_water(posi) {
None
} else {
// 0 to 1, hopefully.
Some(
(gen_ctx.humid_nz.get(wposf.div(1024.0).into_array()) as f32)
.add(1.0)
.mul(0.5),
)
}
})
},
)
},
);
let (((alt_no_water, _), (pure_flux, _)), ((temp_base, _), (humid_base, _))) = threadpool
.join(
|| {
threadpool.join(
|| {
uniform_noise(map_size_lg, |posi, _| {
if pure_water(posi) {
None
} else {
// A version of alt that is uniform over *non-water* (or
// land-adjacent water) chunks.
Some(alt[posi] as f32)
}
})
},
|| {
uniform_noise(map_size_lg, |posi, _| {
if pure_water(posi) {
None
} else {
Some(flux_old[posi])
}
})
},
)
},
|| {
threadpool.join(
|| {
uniform_noise(map_size_lg, |posi, wposf| {
if pure_water(posi) {
None
} else {
// -1 to 1.
Some(gen_ctx.temp_nz.get((wposf).into_array()) as f32)
}
})
},
|| {
uniform_noise(map_size_lg, |posi, wposf| {
// Check whether any tiles around this tile are water.
if pure_water(posi) {
None
} else {
// 0 to 1, hopefully.
Some(
(gen_ctx.humid_nz.get(wposf.div(1024.0).into_array())
as f32)
.add(1.0)
.mul(0.5),
)
}
})
},
)
},
);
let gen_cdf = GenCdf {
humid_base,

3
world/src/sim2/mod.rs
View File

@ -1071,6 +1071,7 @@ mod tests {
#[test]
fn test_economy() {
init();
let threadpool = rayon::ThreadPoolBuilder::new().build().unwrap();
info!("init");
let seed = 59686;
let opts = sim::WorldOpts {
@ -1080,7 +1081,7 @@ mod tests {
};
let mut index = crate::index::Index::new(seed);
info!("Index created");
let mut sim = sim::WorldSim::generate(seed, opts);
let mut sim = sim::WorldSim::generate(seed, opts, &threadpool);
info!("World loaded");
let regenerate_input = false;
if regenerate_input {