diff --git a/client/src/lib.rs b/client/src/lib.rs
index a4e408526a..6cd11b307b 100644
--- a/client/src/lib.rs
+++ b/client/src/lib.rs
@@ -39,7 +39,7 @@ use common::{
     mounting::Rider,
     outcome::Outcome,
     recipe::{ComponentRecipeBook, RecipeBook},
-    resources::{DeltaTime, GameMode, PlayerEntity, ServerTime, Time, TimeOfDay},
+    resources::{DeltaTime, GameMode, MonotonicTime, PlayerEntity, Time, TimeOfDay},
     spiral::Spiral2d,
     terrain::{
         block::Block, map::MapConfig, neighbors, site::DungeonKindMeta, BiomeKind,
@@ -1684,9 +1684,9 @@ impl Client {
         // significant changes to this code. Here is the approximate order of
         // things. Please update it as this code changes.
         //
-        // 1) Collect input from the frontend, apply input effects to the state
+        // 1) Handle messages from the server
+        // 2) Collect input from the frontend, apply input effects to the state
         //    of the game
-        // 2) Handle messages from the server
         // 3) Go through any events (timer-driven or otherwise) that need handling
         //    and apply them to the state of the game
         // 4) Perform a single LocalState tick (i.e: update the world and entities
@@ -1697,33 +1697,7 @@ impl Client {
         // 7) Finish the tick, passing actions of the main thread back
         //    to the frontend
 
-        // 1) Handle input from frontend.
-        // Pass character actions from frontend input to the player's entity.
-        if self.presence.is_some() {
-            prof_span!("handle and send inputs");
-            self.next_control.inputs = inputs;
-            let con = std::mem::take(&mut self.next_control);
-            let time = Duration::from_secs_f64(self.state.ecs().read_resource::<Time>().0);
-            let rcon = self.local_command_gen.gen(time, con);
-            let commands = self
-                .state
-                .ecs()
-                .write_storage::<RemoteController>()
-                .entry(self.entity())
-                .map(|rc| {
-                    let rc = rc.or_insert_with(RemoteController::default);
-                    rc.push(rcon);
-                    rc.commands().clone()
-                });
-            match commands {
-                Ok(commands) => self.send_msg_err(ClientGeneral::Control(commands))?,
-                Err(e) => {
-                    error!(?e, "couldn't create RemoteController for own entity");
-                },
-            };
-        }
-
-        // 2) Build up a list of events for this frame, to be passed to the frontend.
+        // 1) Build up a list of events for this frame, to be passed to the frontend.
         let mut frontend_events = Vec::new();
 
         // Prepare for new events
@@ -1754,6 +1728,34 @@ impl Client {
         // Handle new messages from the server.
         frontend_events.append(&mut self.handle_new_messages()?);
 
+        // 2) Handle input from frontend.
+        // Pass character actions from frontend input to the player's entity.
+        if self.presence.is_some() {
+            prof_span!("handle and send inputs");
+            self.next_control.inputs = inputs;
+            let con = std::mem::take(&mut self.next_control);
+            let time = Duration::from_secs_f64(self.state.ecs().read_resource::<Time>().0);
+            let monotonic_time =
+                Duration::from_secs_f64(self.state.ecs().read_resource::<MonotonicTime>().0);
+            let rcon = self.local_command_gen.gen(time, con);
+            let commands = self
+                .state
+                .ecs()
+                .write_storage::<RemoteController>()
+                .entry(self.entity())
+                .map(|rc| {
+                    let rc = rc.or_insert_with(RemoteController::default);
+                    rc.push(rcon);
+                    rc.prepare_commands(monotonic_time)
+                });
+            match commands {
+                Ok(commands) => self.send_msg_err(ClientGeneral::Control(commands))?,
+                Err(e) => {
+                    error!(?e, "couldn't create RemoteController for own entity");
+                },
+            };
+        }
+
         // 3) Update client local data
         // Check if the invite has timed out and remove if so
         if self
@@ -2155,7 +2157,6 @@ impl Client {
         prof_span!("handle_server_in_game_msg");
         match msg {
             ServerGeneral::TimeSync(time) => {
-                self.state.ecs().write_resource::<ServerTime>().0 = time.0;
                 let dt = self.state.ecs().read_resource::<DeltaTime>().0 as f64;
                 let latency = self
                     .state
@@ -2165,17 +2166,22 @@ impl Client {
                     .map(|rc| rc.avg_latency())
                     .unwrap_or_default();
                 //remove dt as it is applied in state.tick again
-                self.state.ecs().write_resource::<Time>().0 = time.0 + latency.as_secs_f64() - dt;
+                self.state.ecs().write_resource::<Time>().0 = time.0 + latency.as_secs_f64() /* - dt */;
             },
-            ServerGeneral::AckControl(acked_ids, time) => {
+            ServerGeneral::AckControl(acked_ids, _time) => {
                 if let Some(remote_controller) = self
                     .state
                     .ecs()
                     .write_storage::<RemoteController>()
                     .get_mut(self.entity())
                 {
-                    let time = Duration::from_secs_f64(time.0);
-                    remote_controller.acked(acked_ids, time);
+                    // for now ignore the time send by the server as its based on TIME and just use
+                    // MonotonicTime
+                    let monotonic_time = Duration::from_secs_f64(
+                        self.state.ecs().read_resource::<MonotonicTime>().0,
+                    );
+                    //let time = Duration::from_secs_f64(time.0);
+                    remote_controller.acked(acked_ids, monotonic_time);
                     remote_controller.maintain(None);
                 }
             },
diff --git a/common/src/comp/remote_controller.rs b/common/src/comp/remote_controller.rs
index 17fc1d2c29..0264ede3a4 100644
--- a/common/src/comp/remote_controller.rs
+++ b/common/src/comp/remote_controller.rs
@@ -3,6 +3,7 @@ use hashbrown::HashSet;
 use serde::{Deserialize, Serialize};
 use specs::{Component, DenseVecStorage};
 use std::{collections::VecDeque, time::Duration};
+use tracing::warn;
 use vek::Vec3;
 
 pub type ControlCommands = VecDeque<ControlCommand>;
@@ -25,20 +26,23 @@ pub struct RemoteController {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct ControlCommand {
     id: u64,
-    source_time: Duration,
+    /// *Time* when Controller should be applied
+    action_time: Duration,
+    /// *ContinuousMonotonicTime* when this msg was first send to remote
+    first_send_monotonic_time: Option<Duration>,
     msg: Controller,
 }
 
 impl RemoteController {
-    /// delte old and outdated( older than server_time) commands
+    /// delete old and outdated( older than server_time) commands
     pub fn maintain(&mut self, server_time: Option<Duration>) {
         self.commands.make_contiguous();
         if let Some(last) = self.commands.back() {
-            let min_allowed_age = last.source_time;
+            let min_allowed_age = last.action_time;
 
             while let Some(first) = self.commands.front() {
-                if first.source_time + self.max_hold < min_allowed_age
-                    || matches!(server_time, Some(x) if first.source_time < x)
+                if first.action_time + self.max_hold < min_allowed_age
+                    || matches!(server_time, Some(x) if first.action_time < x)
                 {
                     self.commands
                         .pop_front()
@@ -58,7 +62,7 @@ impl RemoteController {
         }
         match self
             .commands
-            .binary_search_by_key(&command.source_time, |e| e.source_time)
+            .binary_search_by_key(&command.action_time, |e| e.action_time)
         {
             Ok(_) => None,
             Err(i) => {
@@ -78,7 +82,7 @@ impl RemoteController {
                 // TODO: improve algorithm to move binary search out of loop
                 if let Err(i) = self
                     .commands
-                    .binary_search_by_key(&command.source_time, |e| e.source_time)
+                    .binary_search_by_key(&command.action_time, |e| e.action_time)
                 {
                     result.insert(id);
                     self.existing_commands.insert(id);
@@ -90,32 +94,42 @@ impl RemoteController {
     }
 
     /// arrived at remote and no longer need to be hold locally
-    pub fn acked(&mut self, ids: HashSet<u64>, server_send_time: Duration) {
-        //get latency from oldest removed command and server_send_time
-        let mut highest_source_time = Duration::from_secs(0);
+    pub fn acked(&mut self, ids: HashSet<u64>, monotonic_time: Duration) {
+        // To get the latency we use the LOWEST `first_send_monotonic_time` in this Set.
+        // Why lowest ? Because we want to be STABLE, and if we just use the highest it
+        // wouldn't be enough latency for all (e.g. retransmittion)
+        let mut lowest_monotonic_time = None;
         self.commands.retain(|c| {
             let retain = !ids.contains(&c.id);
-            if !retain && c.source_time > highest_source_time {
-                highest_source_time = c.source_time;
+            if !retain
+                && c.first_send_monotonic_time.map_or(false, |mt| {
+                    mt < lowest_monotonic_time.unwrap_or(monotonic_time)
+                })
+            {
+                lowest_monotonic_time = c.first_send_monotonic_time;
             }
             retain
         });
-        // we add self.avg_latency here as we must assume that highest_source_time was
-        // increased by avg_latency in client TimeSync though we assume that
-        // avg_latency is quite stable and didnt change much between when the component
-        // was added and now
-        if let Some(latency) =
-            (server_send_time + self.avg_latency).checked_sub(highest_source_time)
-        {
-            common_base::plot!("latency", latency.as_secs_f64());
-            self.avg_latency = (99 * self.avg_latency + latency) / 100;
-        } else {
-            // add 10% and 20ms to the latency
-            self.avg_latency =
-                Duration::from_secs_f64(self.avg_latency.as_secs_f64() * 1.1 + 0.02);
+        if let Some(lowest_monotonic_time) = lowest_monotonic_time {
+            if let Some(latency) = monotonic_time.checked_sub(lowest_monotonic_time) {
+                common_base::plot!("latency", latency.as_secs_f64());
+                self.avg_latency = ((99 * self.avg_latency + latency) / 100).max(latency);
+            } else {
+                warn!(
+                    "latency is negative, this should never be the case as this value is not \
+                     synced and monotonic!"
+                );
+            }
         }
         common_base::plot!("avg_latency", self.avg_latency.as_secs_f64());
-        common_base::plot!("highest_source_time", highest_source_time.as_secs_f64());
+    }
+
+    /// prepare commands for sending
+    pub fn prepare_commands(&mut self, monotonic_time: Duration) -> ControlCommands {
+        for c in &mut self.commands {
+            c.first_send_monotonic_time.get_or_insert(monotonic_time);
+        }
+        self.commands.clone()
     }
 
     pub fn commands(&self) -> &ControlCommands { &self.commands }
@@ -133,7 +147,7 @@ impl RemoteController {
         // compressing 0.8s - 1.2s should lead to index 0-1
         let start_i = match self
             .commands
-            .binary_search_by_key(&start, |e| e.source_time)
+            .binary_search_by_key(&start, |e| e.action_time)
         {
             Ok(i) => i,
             Err(0) => 0,
@@ -141,7 +155,7 @@ impl RemoteController {
         };
         let end_exclusive_i = match self
             .commands
-            .binary_search_by_key(&(start + dt), |e| e.source_time)
+            .binary_search_by_key(&(start + dt), |e| e.action_time)
         {
             Ok(i) => i,
             Err(i) => i,
@@ -159,9 +173,9 @@ impl RemoteController {
         let mut last_start = start;
         for i in start_i..end_exclusive_i {
             let e = &self.commands[i];
-            let local_start = e.source_time.max(last_start);
+            let local_start = e.action_time.max(last_start);
             let local_end = if let Some(e) = self.commands.get(i + 1) {
-                e.source_time.min(start + dt)
+                e.action_time.min(start + dt)
             } else {
                 start + dt
             };
@@ -180,7 +194,7 @@ impl RemoteController {
             // we apply events from all that are started here.
             // if we only are part of 1 tick here we would assume that it was already
             // covered before
-            if i != start_i || e.source_time >= start {
+            if i != start_i || e.action_time >= start {
                 result.actions.append(&mut e.msg.actions.clone());
                 result.events.append(&mut e.msg.events.clone());
                 result
@@ -224,7 +238,7 @@ impl Default for RemoteController {
             commands: VecDeque::new(),
             existing_commands: HashSet::new(),
             max_hold: Duration::from_secs(1),
-            avg_latency: Duration::from_millis(300),
+            avg_latency: Duration::from_millis(50),
         }
     }
 }
@@ -239,10 +253,11 @@ pub struct CommandGenerator {
 }
 
 impl CommandGenerator {
-    pub fn gen(&mut self, time: Duration, msg: Controller) -> ControlCommand {
+    pub fn gen(&mut self, action_time: Duration, msg: Controller) -> ControlCommand {
         self.id += 1;
         ControlCommand {
-            source_time: time,
+            action_time,
+            first_send_monotonic_time: None,
             id: self.id,
             msg,
         }
@@ -252,7 +267,7 @@ impl CommandGenerator {
 impl ControlCommand {
     pub fn msg(&self) -> &Controller { &self.msg }
 
-    pub fn source_time(&self) -> Duration { self.source_time }
+    pub fn source_time(&self) -> Duration { self.action_time }
 }
 
 #[cfg(test)]
@@ -262,7 +277,6 @@ mod tests {
         comp,
         comp::{Climb, ControllerInputs},
     };
-    use std::collections::BTreeMap;
     use vek::{Vec2, Vec3};
 
     const INCREASE: Duration = Duration::from_millis(33);
@@ -358,12 +372,12 @@ mod tests {
         assert_eq!(list.commands[0].id, 1);
         assert_eq!(list.push(data[4].clone()), Some(5));
         assert_eq!(list.commands.len(), 5);
-        list.maintain();
+        list.maintain(None);
         assert_eq!(list.commands.len(), 4);
         assert_eq!(list.commands[0].id, 2);
         assert_eq!(list.push(data[5].clone()), Some(6));
         assert_eq!(list.commands.len(), 5);
-        list.maintain();
+        list.maintain(None);
         assert_eq!(list.commands.len(), 4);
         assert_eq!(list.commands[0].id, 3);
         assert_eq!(list.commands[1].id, 4);
@@ -383,7 +397,7 @@ mod tests {
         let mut to_export = list.commands().iter().map(|e| e.id).collect::<HashSet<_>>();
         // damange one entry
         to_export.remove(&3);
-        list.acked(to_export);
+        list.acked(to_export, Duration::from_secs(6));
         assert_eq!(list.push(data[5].clone()), Some(6));
         assert_eq!(list.push(data[6].clone()), Some(7));
         println!("asd{:?}", &list);
@@ -425,9 +439,14 @@ mod tests {
                 look_dir: Dir::new(Vec3::new(0.0, 1.0, 0.0)),
                 strafing: false,
             },
-            queued_inputs: BTreeMap::new(),
-            events: Vec::new(),
-            actions: Vec::new(),
+            queued_inputs: vec!((comp::InputKind::Jump, comp::InputAttr {
+                select_pos: None,
+                target_entity: None
+            }))
+            .into_iter()
+            .collect(),
+            events: vec!(comp::ControlEvent::EnableLantern),
+            actions: vec!(),
         });
     }
 
@@ -456,6 +475,19 @@ mod tests {
         });
     }
 
+    #[test]
+    fn compress_last_input_afterwards() {
+        let data = generate_control_cmds(4);
+        let mut list = RemoteController::default();
+        list.append(data);
+        let compressed = list.compress(10 * INCREASE, 2 * INCREASE).unwrap();
+        assert_eq!(compressed.inputs, ControllerInputs {
+            move_dir: Vec2::new(0.6, 0.3),
+            move_z: 3.0,
+            ..ControllerInputs::default()
+        });
+    }
+
     #[test]
     fn compress_avg_input_partial() {
         let data = generate_control_cmds(4);
diff --git a/common/src/resources.rs b/common/src/resources.rs
index f068131dfe..80c4dfbe40 100644
--- a/common/src/resources.rs
+++ b/common/src/resources.rs
@@ -10,14 +10,17 @@ use std::ops::{Mul, MulAssign};
 pub struct TimeOfDay(pub f64);
 
 /// A resource that stores the tick (i.e: physics) time.
+/// This will jump on the client in order to simulate network predictions
 #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
 pub struct Time(pub f64);
 
-/// A resource that stores the Server-Time, its needed to calculate the input
-/// delay and adjust Time on the client keep in mind that it contains a
-/// network-latency. Only filled on the client
-#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)]
-pub struct ServerTime(pub f64);
+/// A resource that stores a continuous monotonic time.
+/// It should ONLY EVER be used to be compared to a previous state of
+/// MonotonicTime in order to calculate PING and LATENCIES
+/// DO NOT use it in any game mechanics, use Time or DeltaTime instead
+/// For now this is only needed on the client
+#[derive(Debug, Default)]
+pub struct MonotonicTime(pub f64);
 
 /// A resource that stores the time since the previous tick.
 #[derive(Default)]
diff --git a/common/state/src/state.rs b/common/state/src/state.rs
index 03a97b9cf0..82a8716fc4 100644
--- a/common/state/src/state.rs
+++ b/common/state/src/state.rs
@@ -13,7 +13,7 @@ use common::{
     outcome::Outcome,
     region::RegionMap,
     resources::{
-        DeltaTime, EntitiesDiedLastTick, GameMode, PlayerEntity, ServerTime, Time, TimeOfDay,
+        DeltaTime, EntitiesDiedLastTick, GameMode, MonotonicTime, PlayerEntity, Time, TimeOfDay,
     },
     slowjob::SlowJobPool,
     terrain::{Block, MapSizeLg, TerrainChunk, TerrainGrid},
@@ -256,7 +256,7 @@ impl State {
         ecs.insert(Time(0.0));
 
         // Register unsynced resources used by the ECS.
-        ecs.insert(ServerTime(0.0)); //synced by msg
+        ecs.insert(MonotonicTime(0.0));
         ecs.insert(DeltaTime(0.0));
         ecs.insert(PlayerEntity(None));
         ecs.insert(TerrainGrid::new(map_size_lg, default_chunk).unwrap());
@@ -612,6 +612,7 @@ impl State {
         // Change the time accordingly.
         self.ecs.write_resource::<TimeOfDay>().0 += dt.as_secs_f64() * DAY_CYCLE_FACTOR;
         self.ecs.write_resource::<Time>().0 += dt.as_secs_f64();
+        self.ecs.write_resource::<MonotonicTime>().0 += dt.as_secs_f64();
 
         // Update delta time.
         // Beyond a delta time of MAX_DELTA_TIME, start lagging to avoid skipping