sync - experiment with woobling over multiple ticks, to adjust to small imperfections, later discarded

client/src/lib.rs

@@ -30,7 +30,7 @@ use common::{
         slot::{EquipSlot, InvSlotId, Slot},
         CharacterState, ChatMode, CommandGenerator, ControlAction, ControlEvent, Controller,
         ControllerInputs, GroupManip, InputKind, InventoryAction, InventoryEvent,
-        InventoryUpdateEvent, MapMarkerChange, RemoteController, UtteranceKind,
+        InventoryUpdateEvent, MapMarkerChange, RemoteController, UtteranceKind, Vel,
     },
     event::{EventBus, LocalEvent, UpdateCharacterMetadata},
     grid::Grid,
@@ -238,6 +238,7 @@ pub struct Client {
     local_command_gen: CommandGenerator,
     next_control: Controller,
     inter_tick_rewind_time: Option<Duration>,
+    _rewind_fluctuation_budget: f64,
 
     network: Option<Network>,
     participant: Option<Participant>,
@@ -723,6 +724,7 @@ impl Client {
             local_command_gen: CommandGenerator::default(),
             next_control: Controller::default(),
             inter_tick_rewind_time: None,
+            _rewind_fluctuation_budget: 0.0,
 
             network: Some(network),
             participant: Some(participant),
@@ -1732,7 +1734,9 @@ impl Client {
         frontend_events.append(&mut self.handle_new_messages()?);
 
         // Simulate Ahead
-        if let Some(_rewind_time) = self.inter_tick_rewind_time {
+        common_base::plot!("recived_time_sync", 0.0);
+        if let Some(rewind_time) = self.inter_tick_rewind_time {
+            common_base::plot!("recived_time_sync", 1.0);
             let _time = self.state.ecs().read_resource::<Time>().0 as f64;
             let simulate_ahead = self
                 .state
@@ -1741,10 +1745,49 @@ impl Client {
                 .get(self.entity())
                 .map(|rc| rc.simulate_ahead())
                 .unwrap_or_default();
+            // We substract `dt` here, as otherwise we
+            // Tick1: server_time=100ms dt=60ms ping=30 simulate_ahead=130ms,
+            // rewind_tick=130ms end_tick=100+130+60=290
+            // Tick2: server_time=130ms dt=30ms ping=30 simulate_ahead=130ms,
+            // rewind_tick=130ms end_tick=130+130+30=290
+            // Tick3: server_time=160ms dt=60ms ping=30 simulate_ahead=130ms,
+            // rewind_tick=130ms end_tick=160+130+60=350 with dt substraction
+            // Tick1: server_time=100ms dt=60ms ping=30 simulate_ahead=130ms,
+            // rewind_tick=70ms end_tick=100+70+60=230 Tick2: server_time=130ms
+            // dt=30ms ping=30 simulate_ahead=130ms, rewind_tick=100ms
+            // end_tick=130+100+30=260 Tick3: server_time=160ms dt=60ms ping=30
+            // simulate_ahead=130ms, rewind_tick=70ms end_tick=160+70+60=290
             let simulate_ahead = simulate_ahead.max(dt) - dt;
+            // measurements lead to the effect that smooth_diff is == 0.0 when we add 2
+            // server ticks here.
+            let simulate_ahead = simulate_ahead + Duration::from_secs_f64(1.0 / 30.0);
+
+            let _strict_end_tick_time =
+                simulate_ahead.as_secs_f64() + /*simulated dt of this tick*/dt.as_secs_f64();
+
+            // Simulate_ahead still fluctionates because Server Tick != Client Tick, and we
+            // cant control the phase in which the sync happens.
+            // In order to dampen it, we calculate the smooth_time and make sure to not
+            // derive to much from it
+            let smooth_diff = simulate_ahead.as_secs_f64() - rewind_time.as_secs_f64();
+
+            //const WARP_PERCENT: f64 = 0.05; // make sure we end up not further than 5%
+            // from the estimated tick let warp_budget
+            let simulate_ahead = if smooth_diff / dt.as_secs_f64() > 0.05 {
+                // use
+                simulate_ahead
+            } else {
+                simulate_ahead
+            };
+
+            common_base::plot!("smooth_diff", smooth_diff);
+
+            //let simulate_ahead = simulate_ahead.max(dt)/* - dt*/;
+            //let simulate_ahead = rewind_time.min(simulate_ahead);
             tracing::warn!(?simulate_ahead, ?dt, "simulating ahead again");
+            common_base::plot!("rewind_time", rewind_time.as_secs_f64());
             self.state.rewind_tick(
-                simulate_ahead.max(dt) - dt,
+                simulate_ahead,
                 |dispatch_builder| {
                     add_rewind_systems(dispatch_builder);
                 },
@@ -1752,6 +1795,7 @@ impl Client {
             );
         }
 
+        common_base::plot!("dt", dt.as_secs_f64());
         self.state.tick(
             dt,
             |dispatch_builder| {
@@ -1760,6 +1804,29 @@ impl Client {
             },
             true,
         );
+        let time = self.state.ecs().read_resource::<Time>().0;
+        common_base::plot!("tick_afterwards", time);
+        let vel = self
+            .state
+            .ecs()
+            .read_storage::<Vel>()
+            .get(self.entity())
+            .cloned()
+            .unwrap_or(Vel(Vec3::zero()));
+
+        common_base::plot!("vel_x_after", vel.0.x as f64);
+        common_base::plot!("vel_y_after", vel.0.y as f64);
+        let pos = self
+            .state
+            .ecs()
+            .read_storage::<common::comp::Pos>()
+            .get(self.entity())
+            .cloned()
+            .unwrap_or(common::comp::Pos(Vec3::zero()));
+
+        common_base::plot!("pos_x_after", pos.0.x as f64);
+        common_base::plot!("pos_y_after", pos.0.y as f64);
+        common_base::plot!("pos_z_after", pos.0.z as f64);
 
         // 2) Handle input from frontend.
         // Pass character actions from frontend input to the player's entity.
@@ -2191,6 +2258,8 @@ impl Client {
         prof_span!("handle_server_in_game_msg");
         match msg {
             ServerGeneral::TimeSync(time) => {
+                // Even with a stable network, expect time to oscillate around the actual time
+                // by SERVER_TICK (33.3ms)
                 let old_time = self.state.ecs().read_resource::<Time>().0;
                 let diff = old_time - time.0;
                 self.state.ecs().write_resource::<Time>().0 = time.0;
@@ -2203,20 +2272,28 @@ impl Client {
                     tracing::warn!(?old_time, ?diff, "Time was advanced by server");
                 }
             },
-            ServerGeneral::AckControl(acked_ids, _time) => {
+
+            ServerGeneral::AckControl {
+                acked_ids,
+                highest_ahead_command,
+                predict_available,
+            } => {
                 if let Some(remote_controller) = self
                     .state
                     .ecs()
                     .write_storage::<RemoteController>()
                     .get_mut(self.entity())
                 {
+                    common_base::plot!("server_predict_available", predict_available as f64);
+                    common_base::plot!("highest_ahead_command2", highest_ahead_command);
+
                     // for now ignore the time send by the server as its based on TIME and just use
                     // MonotonicTime
                     let monotonic_time = Duration::from_secs_f64(
                         self.state.ecs().read_resource::<MonotonicTime>().0,
                     );
                     //let time = Duration::from_secs_f64(time.0);
-                    remote_controller.acked(acked_ids, monotonic_time);
+                    remote_controller.acked(acked_ids, monotonic_time, highest_ahead_command);
                     remote_controller.maintain(None);
                 }
             },

common/net/src/msg/ecs_packet.rs

@@ -49,13 +49,13 @@ macro_rules! comp_packet {
                         sync::handle_insert(comp, entity, world);
                     },)*
                     Self::Pos(comp) => {
-                        sync::handle_interp_insert(comp, entity, world, force_update)
+                        sync::handle_interp_insert(comp, entity, world, true)
                     },
                     Self::Vel(comp) => {
-                        sync::handle_interp_insert(comp, entity, world, force_update)
+                        sync::handle_interp_insert(comp, entity, world, true)
                     },
                     Self::Ori(comp) => {
-                        sync::handle_interp_insert(comp, entity, world, force_update)
+                        sync::handle_interp_insert(comp, entity, world, true)
                     },
                 }
             }
@@ -67,13 +67,13 @@ macro_rules! comp_packet {
                         sync::handle_modify(comp, entity, world);
                     },)*
                     Self::Pos(comp) => {
-                        sync::handle_interp_modify(comp, entity, world, force_update)
+                        sync::handle_interp_modify(comp, entity, world, true)
                     },
                     Self::Vel(comp) => {
-                        sync::handle_interp_modify(comp, entity, world, force_update)
+                        sync::handle_interp_modify(comp, entity, world, true)
                     },
                     Self::Ori(comp) => {
-                        sync::handle_interp_modify(comp, entity, world, force_update)
+                        sync::handle_interp_modify(comp, entity, world, true)
                     },
                 }
             }

common/net/src/msg/server.rs

@@ -143,7 +143,14 @@ pub enum ServerGeneral {
     SpectatorSuccess(Vec3<f32>),
     //Ingame related
     TimeSync(Time),
-    AckControl(HashSet<u64>, Time),
+    AckControl {
+        acked_ids: HashSet<u64>,
+        /// measured by the time the furthest command to become active <>
+        /// current server time
+        highest_ahead_command: f64,
+        /// number of predicts available at the server
+        predict_available: usize,
+    },
     GroupUpdate(comp::group::ChangeNotification<Uid>),
     /// Indicate to the client that they are invited to join a group
     Invite {
@@ -318,7 +325,7 @@ impl ServerMsg {
                         //Ingame related
                         ServerGeneral::GroupUpdate(_)
                         | ServerGeneral::TimeSync(_)
-                        | ServerGeneral::AckControl(_, _)
+                        | ServerGeneral::AckControl { .. }
                         | ServerGeneral::Invite { .. }
                         | ServerGeneral::InvitePending(_)
                         | ServerGeneral::InviteComplete { .. }

common/net/src/sync/interpolation.rs

@@ -56,7 +56,7 @@ impl<T: 'static + Send + Sync> Component for InterpBuffer<T> {
 }
 
 // 0 is pure physics, 1 is pure extrapolation
-const PHYSICS_VS_EXTRAPOLATION_FACTOR: f32 = 0.1;
+const PHYSICS_VS_EXTRAPOLATION_FACTOR: f32 = 0.0;
 const POSITION_INTERP_SANITY: Option<f32> = None;
 const VELOCITY_INTERP_SANITY: Option<f32> = None;
 const ENABLE_POSITION_HERMITE: bool = false;

common/src/comp/remote_controller.rs

@@ -3,7 +3,6 @@ use hashbrown::HashSet;
 use serde::{Deserialize, Serialize};
 use specs::{Component, DenseVecStorage};
 use std::{collections::VecDeque, time::Duration};
-use tracing::warn;
 
 pub type ControlCommands = VecDeque<ControlCommand>;
 
@@ -29,6 +28,10 @@ pub struct ControlCommand {
     action_time: Duration,
     /// *ContinuousMonotonicTime* when this msg was first send to remote
     first_send_monotonic_time: Option<Duration>,
+    /// action_time - *Time* when this msg was first send to remote
+    first_send_simulate_ahead_time: Option<f64>,
+    /// *ContinuousMonotonicTime* when this msg was first acked by remote
+    first_acked_monotonic_time: Option<Duration>,
     msg: Controller,
 }
 
@@ -92,23 +95,88 @@ impl RemoteController {
         result
     }
 
-    /// arrived at remote and no longer need to be hold locally
+    /// remote confirmed arrival of commands. (not that they actually were used)
-    pub fn acked(&mut self, ids: HashSet<u64>, monotonic_time: Duration) {
+    /// we need to hold them, (especially with high lag) as we might revert to
-        // To get the latency we use the LOWEST `first_send_monotonic_time` in this Set.
+    /// server-time.
-        // Why lowest ? Because we want to be STABLE, and if we just use the highest it
+    pub fn acked(
-        // wouldn't be enough latency for all (e.g. retransmittion)
+        &mut self,
-        let mut lowest_monotonic_time = None;
+        ids: HashSet<u64>,
-        self.commands.retain(|c| {
+        monotonic_time: Duration,
-            let retain = !ids.contains(&c.id);
+        highest_ahead_command: f64,
-            if !retain
+    ) {
-                && c.first_send_monotonic_time.map_or(false, |mt| {
+        // calculating avg_latency.
-                    mt < lowest_monotonic_time.unwrap_or(monotonic_time)
+        //  - server returns the time the furthest away command has left till it becomes
-                })
+        //    active. As the server time is constant and we stored this value locally,
-            {
+        //    we use it to calculate the first ahead_command for the first command in
-                lowest_monotonic_time = c.first_send_monotonic_time;
+        //    this batch.
+        //  - we try to keep the remote_ahead in a specific window to allow for
+        //    retransmittion and remote tick time
+
+        // find time it took from client -> server
+        let high_filter = self
+            .commands
+            .iter()
+            .filter(|c| ids.contains(&c.id) && c.first_acked_monotonic_time.is_none());
+        if let Some(highest) = high_filter.max_by_key(|c| c.action_time) {
+            let remote_time = highest.action_time.as_secs_f64() - highest_ahead_command;
+            //let latency = highest.first_send_simulate_ahead_time.unwrap_or_default() -
+            // remote_time;
+            common_base::plot!("prob_remote_time", remote_time);
+            //common_base::plot!("prob_avg_latency", latency);
+            //let low_filter = self.commands.iter().filter(|c| ids.contains(&c.id) &&
+            // c.first_send_monotonic_time == highest.first_send_monotonic_time);
+            let low_filter = self
+                .commands
+                .iter()
+                .filter(|c| ids.contains(&c.id) && c.first_acked_monotonic_time.is_none());
+            if let Some(lowest) = low_filter.min_by_key(|c| c.action_time) {
+                let low_high_diff = highest.action_time - lowest.action_time;
+                // if this is 50ms, it means the lowest command arrived at server 50 before it
+                // was used
+                common_base::plot!("highest_ahead_command", highest_ahead_command);
+                let ahead_time = highest_ahead_command - low_high_diff.as_secs_f64();
+                common_base::plot!("low_high_diff", low_high_diff.as_secs_f64());
+                common_base::plot!("ahead_time", ahead_time);
+                const LOWER_END: f64 = 0.18;
+                const UPPER_END: f64 = 0.22;
+                //if ahead_time > 2.0 {
+                let len = self.commands.len();
+                tracing::error!(
+                    ?ahead_time,
+                    ?ids,
+                    ?highest_ahead_command,
+                    ?highest,
+                    ?lowest,
+                    ?len,
+                    "bigger2"
+                );
+                //}
+                if ahead_time < LOWER_END {
+                    self.avg_latency += Duration::from_millis(10);
+                }
+                if ahead_time > UPPER_END && self.avg_latency > Duration::from_millis(3) {
+                    self.avg_latency -= Duration::from_millis(3);
+                }
             }
-            retain
+        }
-        });
+
+        for c in &mut self.commands {
+            c.first_acked_monotonic_time = Some(monotonic_time);
+        }
+
+        /*
+        let mut lowest_monotonic_time = None;
+        for c in &mut self.commands {
+            let new_acked = c.first_acked_monotonic_time.is_none() && ids.contains(&c.id);
+            if new_acked {
+                c.first_acked_monotonic_time = Some(monotonic_time);
+                if c.first_send_monotonic_time.map_or(false, |mt| {
+                    mt < lowest_monotonic_time.unwrap_or(monotonic_time)
+                }) {
+                    lowest_monotonic_time = c.first_send_monotonic_time;
+                }
+            }
+        }
         if let Some(lowest_monotonic_time) = lowest_monotonic_time {
             if let Some(latency) = monotonic_time.checked_sub(lowest_monotonic_time) {
                 common_base::plot!("latency", latency.as_secs_f64());
@@ -119,16 +187,25 @@ impl RemoteController {
                      synced and monotonic!"
                 );
             }
-        }
+        }*/
         common_base::plot!("avg_latency", self.avg_latency.as_secs_f64());
     }
 
     /// prepare commands for sending
+    /// only include commands, that were not yet acked!
     pub fn prepare_commands(&mut self, monotonic_time: Duration) -> ControlCommands {
-        for c in &mut self.commands {
+        let simulate_ahead = self.simulate_ahead().as_secs_f64();
-            c.first_send_monotonic_time.get_or_insert(monotonic_time);
+        self.commands
-        }
+            .iter_mut()
-        self.commands.clone()
+            .filter(|c| c.first_acked_monotonic_time.is_none())
+            .map(|c| {
+                c.first_send_monotonic_time.get_or_insert(monotonic_time);
+                c.first_send_simulate_ahead_time
+                    .get_or_insert(simulate_ahead);
+                &*c
+            })
+            .cloned()
+            .collect()
     }
 
     pub fn commands(&self) -> &ControlCommands { &self.commands }
@@ -230,7 +307,7 @@ impl RemoteController {
     pub fn avg_latency(&self) -> Duration { self.avg_latency }
 
     pub fn simulate_ahead(&self) -> Duration {
-        const FIXED_OFFSET: Duration = Duration::from_millis(50);
+        const FIXED_OFFSET: Duration = Duration::from_millis(0);
         self.avg_latency() + FIXED_OFFSET
     }
 }
@@ -240,7 +317,7 @@ impl Default for RemoteController {
         Self {
             commands: VecDeque::new(),
             existing_commands: HashSet::new(),
-            max_hold: Duration::from_secs(1),
+            max_hold: Duration::from_secs(5),
             avg_latency: Duration::from_millis(50),
         }
     }
@@ -261,6 +338,8 @@ impl CommandGenerator {
         ControlCommand {
             action_time,
             first_send_monotonic_time: None,
+            first_send_simulate_ahead_time: None,
+            first_acked_monotonic_time: None,
             id: self.id,
             msg,
         }

common/state/src/state.rs

@@ -594,15 +594,24 @@ impl State {
     pub fn rewind_tick(
         &mut self,
         simulate_ahead: Duration,
-        add_systems: impl Fn(&mut DispatcherBuilder),
+        add_systems: impl Fn(&mut DispatcherBuilder) + Clone,
         update_terrain_and_regions: bool,
     ) {
+        common_base::plot!("simulate_ahead", simulate_ahead.as_secs_f64());
         let time_of_day = self.ecs.read_resource::<TimeOfDay>().0;
         let _time = self.ecs.read_resource::<Time>().0;
         let monotonic_time = self.ecs.read_resource::<MonotonicTime>().0;
         let delta_time = self.ecs.read_resource::<DeltaTime>().0;
 
-        self.tick(simulate_ahead, add_systems, update_terrain_and_regions);
+        const MAX_INCREMENTS: usize = 100; // The maximum number of collision tests per tick
+        const STEP_SEC: f64 = 0.1;
+        let increments =
+            ((simulate_ahead.as_secs_f64() / STEP_SEC).ceil() as usize).clamped(1, MAX_INCREMENTS);
+        for _i in 0..increments {
+            //tracing::trace!(?i, ?increments, "subtick");
+            let partial = simulate_ahead / (increments as u32);
+            self.tick(partial, add_systems.clone(), update_terrain_and_regions);
+        }
 
         // rewind changes
 
@@ -639,7 +648,7 @@ impl State {
         // Update delta time.
         // Beyond a delta time of MAX_DELTA_TIME, start lagging to avoid skipping
         // important physics events.
-        self.ecs.write_resource::<DeltaTime>().0 = dt.as_secs_f32().min(MAX_DELTA_TIME);
+        self.ecs.write_resource::<DeltaTime>().0 = dt.as_secs_f32(); //.min(MAX_DELTA_TIME);
 
         if update_terrain_and_regions {
             self.update_region_map();

common/systems/src/lib.rs

@@ -22,7 +22,7 @@ use specs::DispatcherBuilder;
 pub fn add_local_systems(dispatch_builder: &mut DispatcherBuilder) {
     dispatch::<predict_controller::Sys>(dispatch_builder, &[]);
     //TODO: don't run interpolation on server
-    dispatch::<interpolation::Sys>(dispatch_builder, &[]);
+    dispatch::<interpolation::Sys>(dispatch_builder, &[&predict_controller::Sys::sys_name()]);
     dispatch::<mount::Sys>(dispatch_builder, &[]);
     dispatch::<controller::Sys>(dispatch_builder, &[&mount::Sys::sys_name()]);
     dispatch::<character_behavior::Sys>(dispatch_builder, &[&controller::Sys::sys_name()]);
@@ -41,8 +41,9 @@ pub fn add_local_systems(dispatch_builder: &mut DispatcherBuilder) {
 }
 
 pub fn add_rewind_systems(dispatch_builder: &mut DispatcherBuilder) {
+    dispatch::<predict_controller::Sys>(dispatch_builder, &[]);
     //TODO: don't run interpolation on server
-    dispatch::<interpolation::Sys>(dispatch_builder, &[]);
+    dispatch::<interpolation::Sys>(dispatch_builder, &[&predict_controller::Sys::sys_name()]);
     dispatch::<mount::Sys>(dispatch_builder, &[]);
     dispatch::<controller::Sys>(dispatch_builder, &[&mount::Sys::sys_name()]);
     dispatch::<character_behavior::Sys>(dispatch_builder, &[&controller::Sys::sys_name()]);

common/systems/src/phys.rs

@@ -216,6 +216,9 @@ impl<'a> PhysicsData<'a> {
             let entity_center = position.0 + Vec3::new(0.0, 0.0, z_min + half_height);
             let flat_radius = collider.bounding_radius() * scale;
             let radius = (flat_radius.powi(2) + half_height.powi(2)).sqrt();
+            //tracing::info!(?i, ?vel, "vel");
+            common_base::plot!("vel_x", vel.0.x as f64);
+            common_base::plot!("vel_y", vel.0.y as f64);
 
             // Move center to the middle between OLD and OLD+VEL_DT
             // so that we can reduce the collision_boundary.

server/src/client.rs

@@ -178,7 +178,7 @@ impl Client {
                     //In-game related
                     ServerGeneral::GroupUpdate(_)
                     | ServerGeneral::TimeSync(_)
-                    | ServerGeneral::AckControl(_, _)
+                    | ServerGeneral::AckControl { .. }
                     | ServerGeneral::Invite { .. }
                     | ServerGeneral::InvitePending(_)
                     | ServerGeneral::InviteComplete { .. }

server/src/sys/msg/in_game.rs

@@ -87,11 +87,20 @@ impl Sys {
             },
             ClientGeneral::Control(rc) => {
                 if presence.kind.controlling_char() {
-                    let ids = remote_controller.append(rc);
+                    let highest_action_time = rc.iter().map(|rc| rc.source_time()).max().unwrap_or_default();
-                    remote_controller.maintain(Some(Duration::from_secs_f64(time.0)));
+                    if rc.is_empty() {
-
+                        tracing::warn!("client send a empty ClientGeneral::Control msg");
-                    // confirm controls
+                    } else {
-                    client.send(ServerGeneral::AckControl(ids, *time))?;
+                        let acked_ids = remote_controller.append(rc);
+                        remote_controller.maintain(Some(Duration::from_secs_f64(time.0)));
+                        let highest_ahead_command = highest_action_time.as_secs_f64()-time.0;
+                        let predict_available = remote_controller.commands().len();
+                        client.send(ServerGeneral::AckControl{
+                            acked_ids,
+                            highest_ahead_command,
+                            predict_available
+                        })?;
+                    }
                     //Todo: FIXME!!!
                     /*
                                                         // Skip respawn if client entity is alive