diff --git a/client/src/lib.rs b/client/src/lib.rs
index c0c90dd2de..bbaa7b75e5 100644
--- a/client/src/lib.rs
+++ b/client/src/lib.rs
@@ -39,7 +39,7 @@ use common::{
     mounting::Rider,
     outcome::Outcome,
     recipe::{ComponentRecipeBook, RecipeBook},
-    resources::{GameMode, PlayerEntity, TimeOfDay},
+    resources::{GameMode, PlayerEntity, ServerTime, Time, TimeOfDay},
     spiral::Spiral2d,
     terrain::{
         block::Block, map::MapConfig, neighbors, site::DungeonKindMeta, BiomeKind,
@@ -1683,7 +1683,6 @@ impl Client {
         &mut self,
         inputs: ControllerInputs,
         dt: Duration,
-        total_tick_time: Duration,
         add_foreign_systems: impl Fn(&mut DispatcherBuilder),
     ) -> Result<Vec<Event>, Error> {
         span!(_guard, "tick", "Client::tick");
@@ -1712,7 +1711,8 @@ impl Client {
             prof_span!("handle and send inputs");
             self.next_control.inputs = inputs;
             let con = std::mem::take(&mut self.next_control);
-            let rcon = self.local_command_gen.gen(total_tick_time, con);
+            let time = Duration::from_secs_f64(self.state.ecs().read_resource::<Time>().0);
+            let rcon = self.local_command_gen.gen(time, con);
             let commands = self
                 .state
                 .ecs()
@@ -2178,15 +2178,27 @@ impl Client {
     ) -> Result<(), Error> {
         prof_span!("handle_server_in_game_msg");
         match msg {
-            ServerGeneral::AckControl(acked_ids) => {
+            ServerGeneral::TimeSync(time) => {
+                self.state.ecs().write_resource::<ServerTime>().0 = time.0;
+                let latency = self
+                    .state
+                    .ecs()
+                    .read_storage::<RemoteController>()
+                    .get(self.entity())
+                    .map(|rc| rc.avg_latency())
+                    .unwrap_or_default();
+                self.state.ecs().write_resource::<Time>().0 = time.0 + latency.as_secs_f64();
+            },
+            ServerGeneral::AckControl(acked_ids, time) => {
                 if let Some(remote_controller) = self
                     .state
                     .ecs()
                     .write_storage::<RemoteController>()
                     .get_mut(self.entity())
                 {
-                    remote_controller.acked(acked_ids);
-                    remote_controller.maintain();
+                    let time = Duration::from_secs_f64(time.0);
+                    remote_controller.acked(acked_ids, time);
+                    remote_controller.maintain(None);
                 }
             },
             ServerGeneral::GroupUpdate(change_notification) => {
@@ -2870,12 +2882,8 @@ mod tests {
             let mut clock = Clock::new(Duration::from_secs_f64(SPT));
 
             //tick
-            let events_result: Result<Vec<Event>, Error> = client.tick(
-                comp::ControllerInputs::default(),
-                clock.dt(),
-                clock.total_tick_time(),
-                |_| {},
-            );
+            let events_result: Result<Vec<Event>, Error> =
+                client.tick(comp::ControllerInputs::default(), clock.dt(), |_| {});
 
             //chat functionality
             client.send_chat("foobar".to_string());
diff --git a/common/net/src/msg/server.rs b/common/net/src/msg/server.rs
index 8e4b405858..e0cfa2bdef 100644
--- a/common/net/src/msg/server.rs
+++ b/common/net/src/msg/server.rs
@@ -142,7 +142,8 @@ pub enum ServerGeneral {
     CharacterSuccess,
     SpectatorSuccess(Vec3<f32>),
     //Ingame related
-    AckControl(HashSet<u64>),
+    TimeSync(Time),
+    AckControl(HashSet<u64>, Time),
     GroupUpdate(comp::group::ChangeNotification<Uid>),
     /// Indicate to the client that they are invited to join a group
     Invite {
@@ -316,7 +317,8 @@ impl ServerMsg {
                         },
                         //Ingame related
                         ServerGeneral::GroupUpdate(_)
-                        | ServerGeneral::AckControl(_)
+                        | ServerGeneral::TimeSync(_)
+                        | ServerGeneral::AckControl(_, _)
                         | ServerGeneral::Invite { .. }
                         | ServerGeneral::InvitePending(_)
                         | ServerGeneral::InviteComplete { .. }
diff --git a/common/src/comp/remote_controller.rs b/common/src/comp/remote_controller.rs
index 983414bd3a..ae4e5f3688 100644
--- a/common/src/comp/remote_controller.rs
+++ b/common/src/comp/remote_controller.rs
@@ -19,6 +19,7 @@ pub struct RemoteController {
     commands: ControlCommands,
     existing_commands: HashSet<u64>,
     max_hold: Duration,
+    avg_latency: Duration,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -29,14 +30,16 @@ pub struct ControlCommand {
 }
 
 impl RemoteController {
-    /// delte old commands
-    pub fn maintain(&mut self) {
+    /// delte old and outdated( older than server_time) commands
+    pub fn maintain(&mut self, server_time: Option<Duration>) {
         self.commands.make_contiguous();
         if let Some(last) = self.commands.back() {
             let min_allowed_age = last.source_time;
 
             while let Some(first) = self.commands.front() {
-                if first.source_time + self.max_hold < min_allowed_age {
+                if first.source_time + self.max_hold < min_allowed_age
+                    || matches!(server_time, Some(x) if first.source_time < x)
+                {
                     self.commands
                         .pop_front()
                         .map(|c| self.existing_commands.remove(&c.id));
@@ -87,7 +90,32 @@ impl RemoteController {
     }
 
     /// arrived at remote and no longer need to be hold locally
-    pub fn acked(&mut self, ids: HashSet<u64>) { self.commands.retain(|c| !ids.contains(&c.id)); }
+    pub fn acked(&mut self, ids: HashSet<u64>, server_send_time: Duration) {
+        //get latency from oldest removed command and server_send_time
+        let mut highest_source_time = Duration::from_secs(0);
+        self.commands.retain(|c| {
+            let retain = !ids.contains(&c.id);
+            if !retain && c.source_time > highest_source_time {
+                highest_source_time = c.source_time;
+            }
+            retain
+        });
+        // we add self.avg_latency here as we must assume that highest_source_time was
+        // increased by avg_latency in client TimeSync though we assume that
+        // avg_latency is quite stable and didnt change much between when the component
+        // was added and now
+        if let Some(latency) =
+            (server_send_time + self.avg_latency).checked_sub(highest_source_time)
+        {
+            tracing::error!(?latency, "ggg");
+            self.avg_latency = (99 * self.avg_latency + latency) / 100;
+        } else {
+            // add 5% and 10ms to the latency
+            self.avg_latency =
+                Duration::from_secs_f64(self.avg_latency.as_secs_f64() * 1.05 + 0.01);
+        }
+        tracing::error!(?highest_source_time, ?self.avg_latency, "aaa");
+    }
 
     pub fn commands(&self) -> &ControlCommands { &self.commands }
 
@@ -117,9 +145,6 @@ impl RemoteController {
             Ok(i) => i,
             Err(i) => i,
         };
-        println!("start_i {:?}", start_i);
-        println!("end_exclusive_i {:?}", end_exclusive_i);
-
         if self.commands.is_empty() || end_exclusive_i == start_i {
             return None;
         }
@@ -140,8 +165,6 @@ impl RemoteController {
                 start + dt
             };
             let local_dur = local_end - local_start;
-            println!("local_start {:?}, local_end {:?}", local_start, local_end);
-            println!("source_time {:?}", e.source_time);
             result.inputs.move_dir += e.msg.inputs.move_dir * local_dur.as_secs_f32();
             result.inputs.move_z += e.msg.inputs.move_z * local_dur.as_secs_f32();
             look_dir = result.inputs.look_dir.to_vec()
@@ -153,6 +176,16 @@ impl RemoteController {
                 .break_block_pos
                 .or(e.msg.inputs.break_block_pos);
             result.inputs.strafing = result.inputs.strafing || e.msg.inputs.strafing;
+            // we apply events from all that are started here.
+            // if we only are part of 1 tick here we would assume that it was already
+            // covered before
+            if i != start_i || e.source_time >= start {
+                result.actions.append(&mut e.msg.actions.clone());
+                result.events.append(&mut e.msg.events.clone());
+                result
+                    .queued_inputs
+                    .append(&mut e.msg.queued_inputs.clone());
+            }
             last_start = local_start;
         }
         result.inputs.move_dir /= dt.as_secs_f32();
@@ -161,6 +194,13 @@ impl RemoteController {
 
         Some(result)
     }
+
+    /// the average latency is 300ms by default and will be adjusted over time
+    /// with every Ack the server sends its local tick time at the Ack
+    /// so we can calculate how much time was spent for 1 way from
+    /// server->client and assume that this is also true for client->server
+    /// latency
+    pub fn avg_latency(&self) -> Duration { self.avg_latency }
 }
 
 impl Default for RemoteController {
@@ -169,6 +209,7 @@ impl Default for RemoteController {
             commands: VecDeque::new(),
             existing_commands: HashSet::new(),
             max_hold: Duration::from_secs(1),
+            avg_latency: Duration::from_millis(300),
         }
     }
 }
diff --git a/common/src/resources.rs b/common/src/resources.rs
index 4722c4e8b4..57813341fd 100644
--- a/common/src/resources.rs
+++ b/common/src/resources.rs
@@ -13,6 +13,12 @@ pub struct TimeOfDay(pub f64);
 #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
 pub struct Time(pub f64);
 
+/// A resource that stores the Server-Time, its needed to calculate the input
+/// delay and adjust Time on the client keep in mind that it contains a
+/// network-latency. Only filled on the client
+#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)]
+pub struct ServerTime(pub f64);
+
 /// A resource that stores the time since the previous tick.
 #[derive(Default)]
 pub struct DeltaTime(pub f32);
diff --git a/common/state/src/state.rs b/common/state/src/state.rs
index aa2ee521cb..ece16024ce 100644
--- a/common/state/src/state.rs
+++ b/common/state/src/state.rs
@@ -13,8 +13,8 @@ use common::{
     outcome::Outcome,
     region::RegionMap,
     resources::{
-        DeltaTime, EntitiesDiedLastTick, GameMode, PlayerEntity, PlayerPhysicsSettings, Time,
-        TimeOfDay,
+        DeltaTime, EntitiesDiedLastTick, GameMode, PlayerEntity, PlayerPhysicsSettings, ServerTime,
+        Time, TimeOfDay,
     },
     slowjob::SlowJobPool,
     terrain::{Block, MapSizeLg, TerrainChunk, TerrainGrid},
@@ -257,6 +257,7 @@ impl State {
         ecs.insert(Time(0.0));
 
         // Register unsynced resources used by the ECS.
+        ecs.insert(ServerTime(0.0)); //synced by msg
         ecs.insert(DeltaTime(0.0));
         ecs.insert(PlayerEntity(None));
         ecs.insert(TerrainGrid::new(map_size_lg, default_chunk).unwrap());
diff --git a/common/systems/src/predict_controller.rs b/common/systems/src/predict_controller.rs
index a7d2014266..ab4fb48175 100644
--- a/common/systems/src/predict_controller.rs
+++ b/common/systems/src/predict_controller.rs
@@ -1,6 +1,6 @@
 use common::{
     comp::{Controller, RemoteController},
-    resources::{Time, DeltaTime,},
+    resources::{DeltaTime, Time},
 };
 use common_ecs::{Job, Origin, Phase, System};
 use specs::{
diff --git a/server/src/client.rs b/server/src/client.rs
index 286d00ea5a..c0afacefee 100644
--- a/server/src/client.rs
+++ b/server/src/client.rs
@@ -103,7 +103,8 @@ impl Client {
                     },
                     //In-game related
                     ServerGeneral::GroupUpdate(_)
-                    | ServerGeneral::AckControl(_)
+                    | ServerGeneral::TimeSync(_)
+                    | ServerGeneral::AckControl(_, _)
                     | ServerGeneral::Invite { .. }
                     | ServerGeneral::InvitePending(_)
                     | ServerGeneral::InviteComplete { .. }
@@ -176,7 +177,8 @@ impl Client {
                     },
                     //In-game related
                     ServerGeneral::GroupUpdate(_)
-                    | ServerGeneral::AckControl(_)
+                    | ServerGeneral::TimeSync(_)
+                    | ServerGeneral::AckControl(_, _)
                     | ServerGeneral::Invite { .. }
                     | ServerGeneral::InvitePending(_)
                     | ServerGeneral::InviteComplete { .. }
diff --git a/server/src/sys/msg/in_game.rs b/server/src/sys/msg/in_game.rs
index ee8bc0c3bd..b121ef9509 100644
--- a/server/src/sys/msg/in_game.rs
+++ b/server/src/sys/msg/in_game.rs
@@ -9,7 +9,7 @@ use common::{
     event::{EventBus, ServerEvent},
     link::Is,
     mounting::Rider,
-    resources::{PlayerPhysicsSetting, PlayerPhysicsSettings},
+    resources::{PlayerPhysicsSetting, PlayerPhysicsSettings, Time},
     slowjob::SlowJobPool,
     terrain::TerrainGrid,
     vol::ReadVol,
@@ -20,7 +20,7 @@ use common_state::{BlockChange, BuildAreas};
 use core::mem;
 use rayon::prelude::*;
 use specs::{Entities, Join, Read, ReadExpect, ReadStorage, Write, WriteStorage};
-use std::{borrow::Cow, time::Instant};
+use std::{borrow::Cow, time::Instant, time::Duration};
 use tracing::{debug, trace, warn};
 use vek::*;
 
@@ -47,6 +47,7 @@ impl Sys {
     fn handle_client_in_game_msg(
         server_emitter: &mut common::event::Emitter<'_, ServerEvent>,
         entity: specs::Entity,
+        time: &Time,
         client: &Client,
         maybe_presence: &mut Option<&mut Presence>,
         terrain: &ReadExpect<'_, TerrainGrid>,
@@ -96,9 +97,10 @@ impl Sys {
             ClientGeneral::Control(rc) => {
                 if presence.kind.controlling_char() {
                     let ids = remote_controller.append(rc);
-                    remote_controller.maintain();
+                    remote_controller.maintain(Some(Duration::from_secs_f64(time.0)));
+
                     // confirm controls
-                    client.send(ServerGeneral::AckControl(ids))?;
+                    client.send(ServerGeneral::AckControl(ids, *time))?;
                     //Todo: FIXME!!!
                     /*
                                                         // Skip respawn if client entity is alive
@@ -316,6 +318,7 @@ impl<'a> System<'a> for Sys {
     #[allow(clippy::type_complexity)]
     type SystemData = (
         Entities<'a>,
+        Read<'a, Time>,
         Read<'a, EventBus<ServerEvent>>,
         ReadExpect<'a, TerrainGrid>,
         ReadExpect<'a, SlowJobPool>,
@@ -347,6 +350,7 @@ impl<'a> System<'a> for Sys {
         _job: &mut Job<Self>,
         (
             entities,
+            time,
             server_event_bus,
             terrain,
             slow_jobs,
@@ -433,6 +437,7 @@ impl<'a> System<'a> for Sys {
                         Self::handle_client_in_game_msg(
                             server_emitter,
                             entity,
+                            &time,
                             client,
                             &mut clearable_maybe_presence,
                             &terrain,
diff --git a/server/src/sys/msg/ping.rs b/server/src/sys/msg/ping.rs
index cf56910326..0a5a7ecde0 100644
--- a/server/src/sys/msg/ping.rs
+++ b/server/src/sys/msg/ping.rs
@@ -5,6 +5,7 @@ use common::{
 };
 use common_ecs::{Job, Origin, Phase, System};
 use common_net::msg::PingMsg;
+use common_net::msg::ServerGeneral;
 use rayon::prelude::*;
 use specs::{Entities, ParJoin, Read, WriteStorage};
 use tracing::{debug, info};
@@ -47,8 +48,10 @@ impl<'a> System<'a> for Sys {
                     client.participant.as_mut().map(|p| p.try_fetch_event())
                 {}
 
+                client.send_fallible(ServerGeneral::TimeSync(*time));
                 let res = super::try_recv_all(client, 4, Self::handle_ping_msg);
 
+
                 match res {
                     Err(e) => {
                         debug!(?entity, ?e, "network error with client, disconnecting");
diff --git a/voxygen/src/menu/char_selection/mod.rs b/voxygen/src/menu/char_selection/mod.rs
index f54cdf67ee..a024fa8f5c 100644
--- a/voxygen/src/menu/char_selection/mod.rs
+++ b/voxygen/src/menu/char_selection/mod.rs
@@ -208,7 +208,6 @@ impl PlayState for CharSelectionState {
             let res = self.client.borrow_mut().tick(
                 comp::ControllerInputs::default(),
                 global_state.clock.dt(),
-                global_state.clock.total_tick_time(),
                 |_| {},
             );
             match res {
diff --git a/voxygen/src/menu/main/mod.rs b/voxygen/src/menu/main/mod.rs
index 722ec2a1d9..5cd3201780 100644
--- a/voxygen/src/menu/main/mod.rs
+++ b/voxygen/src/menu/main/mod.rs
@@ -226,7 +226,6 @@ impl PlayState for MainMenuState {
             match client.tick(
                 comp::ControllerInputs::default(),
                 global_state.clock.dt(),
-                global_state.clock.total_tick_time(),
                 |_| {},
             ) {
                 Ok(events) => {
diff --git a/voxygen/src/session/mod.rs b/voxygen/src/session/mod.rs
index 0ba5898baf..073728b03b 100644
--- a/voxygen/src/session/mod.rs
+++ b/voxygen/src/session/mod.rs
@@ -270,12 +270,7 @@ impl SessionState {
             self.mumble_link.update(player_pos, player_pos);
         }
 
-        for event in client.tick(
-            self.inputs.clone(),
-            dt,
-            global_state.clock.total_tick_time(),
-            crate::ecs::sys::add_local_systems,
-        )? {
+        for event in client.tick(self.inputs.clone(), dt, crate::ecs::sys::add_local_systems)? {
             match event {
                 client::Event::Chat(m) => {
                     self.hud.new_message(m);