diff --git a/Cargo.lock b/Cargo.lock
index 008be4ba06..385f69d304 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5607,6 +5607,7 @@ version = "0.9.0"
 dependencies = [
  "bincode",
  "hashbrown",
+ "num_cpus",
  "rayon",
  "scopeguard",
  "serde",
@@ -5766,6 +5767,7 @@ dependencies = [
  "clap",
  "crossterm 0.19.0",
  "lazy_static",
+ "num_cpus",
  "ron",
  "serde",
  "signal-hook 0.3.8",
diff --git a/common/state/Cargo.toml b/common/state/Cargo.toml
index 26c2cec587..74b42220ce 100644
--- a/common/state/Cargo.toml
+++ b/common/state/Cargo.toml
@@ -18,6 +18,7 @@ common-ecs = { package = "veloren-common-ecs", path = "../ecs" }
 common-base = { package = "veloren-common-base", path = "../base" }
 
 rayon = "1.5"
+num_cpus = "1.0"
 tracing = { version = "0.1", default-features = false }
 vek = { version = "=0.14.1", features = ["serde"] }
 
diff --git a/common/state/src/state.rs b/common/state/src/state.rs
index f34771f1d2..3819ad200f 100644
--- a/common/state/src/state.rs
+++ b/common/state/src/state.rs
@@ -103,6 +103,9 @@ impl State {
 
         let thread_pool = Arc::new(
             ThreadPoolBuilder::new()
+                .num_threads(
+                    num_cpus::get().max(2), /* Have AT LEAST 2 rayon threads */
+                )
                 .thread_name(move |i| format!("rayon-{}-{}", thread_name_infix, i))
                 .build()
                 .unwrap(),
@@ -208,7 +211,7 @@ impl State {
         ecs.insert(Vec::<common::outcome::Outcome>::new());
         ecs.insert(common::CachedSpatialGrid::default());
 
-        let slow_limit = thread_pool.current_num_threads().max(2) as u64;
+        let slow_limit = num_cpus::get().max(2) as u64;
         let slow_limit = slow_limit / 2 + slow_limit / 4;
         tracing::trace!(?slow_limit, "Slow Thread limit");
         ecs.insert(SlowJobPool::new(slow_limit, Arc::clone(&thread_pool)));
diff --git a/server-cli/Cargo.toml b/server-cli/Cargo.toml
index 6041be5e66..59217755b3 100644
--- a/server-cli/Cargo.toml
+++ b/server-cli/Cargo.toml
@@ -28,6 +28,7 @@ common-net = { package = "veloren-common-net", path = "../common/net" }
 common-frontend = { package = "veloren-common-frontend", path = "../common/frontend" }
 
 tokio = { version = "1", default-features = false, features = ["rt-multi-thread"] }
+num_cpus = "1.0"
 ansi-parser = "0.7"
 clap = "2.33"
 crossterm = "0.19"
diff --git a/server-cli/src/main.rs b/server-cli/src/main.rs
index 39dbe97a9f..fb75296802 100644
--- a/server-cli/src/main.rs
+++ b/server-cli/src/main.rs
@@ -117,9 +117,12 @@ fn main() -> io::Result<()> {
         path
     };
 
+    // We don't need that many threads in the async pool, at least 2 but generally
+    // 25% of all available will do
     let runtime = Arc::new(
         tokio::runtime::Builder::new_multi_thread()
             .enable_all()
+            .worker_threads((num_cpus::get() / 4).max(2))
             .thread_name_fn(|| {
                 static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
                 let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
diff --git a/voxygen/src/menu/main/client_init.rs b/voxygen/src/menu/main/client_init.rs
index 6cbe4ee2bc..9d4fc61504 100644
--- a/voxygen/src/menu/main/client_init.rs
+++ b/voxygen/src/menu/main/client_init.rs
@@ -66,7 +66,7 @@ impl ClientInit {
             Arc::new(
                 runtime::Builder::new_multi_thread()
                     .enable_all()
-                    .worker_threads(if cores > 4 { cores - 1 } else { cores })
+                    .worker_threads((cores / 4).max(2))
                     .thread_name_fn(|| {
                         static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
                         let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
diff --git a/voxygen/src/singleplayer.rs b/voxygen/src/singleplayer.rs
index 8cd06499e0..68252fa2dd 100644
--- a/voxygen/src/singleplayer.rs
+++ b/voxygen/src/singleplayer.rs
@@ -87,7 +87,7 @@ impl Singleplayer {
         let runtime = Arc::new(
             tokio::runtime::Builder::new_multi_thread()
                 .enable_all()
-                .worker_threads(if cores > 4 { cores - 1 } else { cores })
+                .worker_threads((cores / 4).max(2))
                 .thread_name_fn(|| {
                     static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
                     let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);