From 608a08eb7616e20cd2404653cb2dc6109297213c Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 2 Nov 2022 10:21:10 +0800 Subject: [PATCH 1/9] refactor: md5 of revision --- .../flowy-document/src/editor/document.rs | 6 +- .../flowy-document/src/editor/queue.rs | 2 +- .../rust-lib/flowy-document/src/manager.rs | 9 +-- .../flowy-document/src/old_editor/queue.rs | 28 ++++----- .../src/old_editor/web_socket.rs | 7 +-- .../flowy-document/src/services/migration.rs | 8 +-- .../rev_sqlite/document_rev_sqlite_v0.rs | 20 +++--- .../rev_sqlite/document_rev_sqlite_v1.rs | 20 +++--- .../src/services/persistence/migration.rs | 3 +- .../src/services/persistence/mod.rs | 4 +- .../rev_sqlite/folder_rev_sqlite.rs | 20 +++--- .../flowy-folder/src/services/web_socket.rs | 8 +-- frontend/rust-lib/flowy-grid/src/manager.rs | 23 +++---- .../persistence/rev_sqlite/grid_block_impl.rs | 20 +++--- .../persistence/rev_sqlite/grid_impl.rs | 20 +++--- .../persistence/rev_sqlite/grid_view_impl.rs | 20 +++--- .../rust-lib/flowy-revision/src/cache/disk.rs | 25 ++++---- .../flowy-revision/src/cache/memory.rs | 42 ++++++------- .../flowy-revision/src/cache/reset.rs | 4 +- .../flowy-revision/src/conflict_resolve.rs | 15 ++--- .../flowy-revision/src/rev_manager.rs | 63 +++++++++++++++++-- .../flowy-revision/src/rev_persistence.rs | 28 +++++---- .../rust-lib/flowy-revision/src/ws_manager.rs | 2 +- .../rust-lib/flowy-revision/tests/main.rs | 1 + .../flowy-sdk/src/deps_resolve/folder_deps.rs | 15 +++-- .../src/client_document/document_pad.rs | 5 +- .../src/client_folder/folder_pad.rs | 12 ++-- .../src/client_grid/block_revision_pad.rs | 4 +- .../src/client_grid/grid_revision_pad.rs | 8 +-- .../src/client_grid/view_revision_pad.rs | 4 +- .../flowy-sync/src/entities/revision.rs | 57 ++++++++--------- shared-lib/flowy-sync/src/util.rs | 3 +- shared-lib/lib-ot/src/core/node_tree/tree.rs | 16 ++++- shared-lib/lib-ot/tests/node/serde_test.rs | 13 +++- 34 files changed, 300 insertions(+), 235 deletions(-) create mode 100644 frontend/rust-lib/flowy-revision/tests/main.rs diff --git a/frontend/rust-lib/flowy-document/src/editor/document.rs b/frontend/rust-lib/flowy-document/src/editor/document.rs index bf3ff469c8..8af53a5727 100644 --- a/frontend/rust-lib/flowy-document/src/editor/document.rs +++ b/frontend/rust-lib/flowy-document/src/editor/document.rs @@ -28,9 +28,9 @@ impl Document { } } - pub fn md5(&self) -> String { - // format!("{:x}", md5::compute(bytes)) - "".to_owned() + pub fn document_md5(&self) -> String { + let bytes = self.tree.to_bytes(); + format!("{:x}", md5::compute(&bytes)) } pub fn get_tree(&self) -> &NodeTree { diff --git a/frontend/rust-lib/flowy-document/src/editor/queue.rs b/frontend/rust-lib/flowy-document/src/editor/queue.rs index ef415cb4f1..24448c9bca 100644 --- a/frontend/rust-lib/flowy-document/src/editor/queue.rs +++ b/frontend/rust-lib/flowy-document/src/editor/queue.rs @@ -63,7 +63,7 @@ impl DocumentQueue { Command::ComposeTransaction { transaction, ret } => { self.document.write().await.apply_transaction(transaction.clone())?; let _ = self - .save_local_operations(transaction, self.document.read().await.md5()) + .save_local_operations(transaction, self.document.read().await.document_md5()) .await?; let _ = ret.send(Ok(())); } diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index ac031ad35b..658e3e0a99 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -12,11 +12,8 @@ use flowy_revision::{ RevisionCloudService, RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence, }; use flowy_sync::client_document::initial_delta_document_content; -use flowy_sync::entities::{ - document::DocumentIdPB, - revision::{md5, RepeatedRevision, Revision}, - ws_data::ServerRevisionWSData, -}; +use flowy_sync::entities::{document::DocumentIdPB, revision::Revision, ws_data::ServerRevisionWSData}; +use flowy_sync::util::md5; use lib_infra::future::FutureResult; use lib_ws::WSConnectState; use std::any::Any; @@ -139,7 +136,7 @@ impl DocumentManager { Ok(()) } - pub async fn create_document>(&self, doc_id: T, revisions: RepeatedRevision) -> FlowyResult<()> { + pub async fn create_document>(&self, doc_id: T, revisions: Vec) -> FlowyResult<()> { let doc_id = doc_id.as_ref().to_owned(); let db_pool = self.persistence.database.db_pool()?; // Maybe we could save the document to disk without creating the RevisionManager diff --git a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs index bfe7336257..063ae4f254 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs @@ -3,7 +3,7 @@ use crate::DocumentUser; use async_stream::stream; use flowy_database::ConnectionPool; use flowy_error::FlowyError; -use flowy_revision::{OperationsMD5, RevisionManager, TransformOperations}; +use flowy_revision::{RevisionMD5, RevisionManager, TransformOperations}; use flowy_sync::{ client_document::{history::UndoResult, ClientDocument}, entities::revision::{RevId, Revision}, @@ -70,7 +70,7 @@ impl EditDocumentQueue { EditorCommand::ComposeLocalOperations { operations, ret } => { let mut document = self.document.write().await; let _ = document.compose_operations(operations.clone())?; - let md5 = document.md5(); + let md5 = document.document_md5(); drop(document); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); @@ -78,16 +78,16 @@ impl EditDocumentQueue { EditorCommand::ComposeRemoteOperation { client_operations, ret } => { let mut document = self.document.write().await; let _ = document.compose_operations(client_operations.clone())?; - let md5 = document.md5(); + let md5 = document.document_md5(); drop(document); - let _ = ret.send(Ok(md5)); + let _ = ret.send(Ok(md5.into())); } EditorCommand::ResetOperations { operations, ret } => { let mut document = self.document.write().await; let _ = document.set_operations(operations); - let md5 = document.md5(); + let md5 = document.document_md5(); drop(document); - let _ = ret.send(Ok(md5)); + let _ = ret.send(Ok(md5.into())); } EditorCommand::TransformOperations { operations, ret } => { let f = || async { @@ -114,14 +114,14 @@ impl EditDocumentQueue { EditorCommand::Insert { index, data, ret } => { let mut write_guard = self.document.write().await; let operations = write_guard.insert(index, data)?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } EditorCommand::Delete { interval, ret } => { let mut write_guard = self.document.write().await; let operations = write_guard.delete(interval)?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } @@ -132,14 +132,14 @@ impl EditDocumentQueue { } => { let mut write_guard = self.document.write().await; let operations = write_guard.format(interval, attribute)?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } EditorCommand::Replace { interval, data, ret } => { let mut write_guard = self.document.write().await; let operations = write_guard.replace(interval, data)?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } @@ -152,14 +152,14 @@ impl EditDocumentQueue { EditorCommand::Undo { ret } => { let mut write_guard = self.document.write().await; let UndoResult { operations } = write_guard.undo()?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } EditorCommand::Redo { ret } => { let mut write_guard = self.document.write().await; let UndoResult { operations } = write_guard.redo()?; - let md5 = write_guard.md5(); + let md5 = write_guard.document_md5(); let _ = self.save_local_operations(operations, md5).await?; let _ = ret.send(Ok(())); } @@ -197,11 +197,11 @@ pub(crate) enum EditorCommand { }, ComposeRemoteOperation { client_operations: DeltaTextOperations, - ret: Ret, + ret: Ret, }, ResetOperations { operations: DeltaTextOperations, - ret: Ret, + ret: Ret, }, TransformOperations { operations: DeltaTextOperations, diff --git a/frontend/rust-lib/flowy-document/src/old_editor/web_socket.rs b/frontend/rust-lib/flowy-document/src/old_editor/web_socket.rs index 9b9a870f1f..bbaa1c511f 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/web_socket.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/web_socket.rs @@ -136,7 +136,7 @@ impl ConflictResolver for DocumentConflictResolv fn compose_operations( &self, operations: DeltaDocumentResolveOperations, - ) -> BoxResultFuture { + ) -> BoxResultFuture { let tx = self.edit_cmd_tx.clone(); let operations = operations.into_inner(); Box::pin(async move { @@ -172,10 +172,7 @@ impl ConflictResolver for DocumentConflictResolv }) } - fn reset_operations( - &self, - operations: DeltaDocumentResolveOperations, - ) -> BoxResultFuture { + fn reset_operations(&self, operations: DeltaDocumentResolveOperations) -> BoxResultFuture { let tx = self.edit_cmd_tx.clone(); let operations = operations.into_inner(); Box::pin(async move { diff --git a/frontend/rust-lib/flowy-document/src/services/migration.rs b/frontend/rust-lib/flowy-document/src/services/migration.rs index 6dc8bed07d..d922486e47 100644 --- a/frontend/rust-lib/flowy-document/src/services/migration.rs +++ b/frontend/rust-lib/flowy-document/src/services/migration.rs @@ -4,9 +4,9 @@ use crate::DocumentDatabase; use bytes::Bytes; use flowy_database::kv::KV; use flowy_error::FlowyResult; -use flowy_revision::disk::{RevisionDiskCache, RevisionRecord}; -use flowy_sync::entities::revision::{md5, Revision}; -use flowy_sync::util::make_operations_from_revisions; +use flowy_revision::disk::{RevisionDiskCache, SyncRecord}; +use flowy_sync::entities::revision::Revision; +use flowy_sync::util::{make_operations_from_revisions, md5}; use std::sync::Arc; const V1_MIGRATION: &str = "DOCUMENT_V1_MIGRATION"; @@ -44,7 +44,7 @@ impl DocumentMigration { let bytes = Bytes::from(transaction.to_bytes()?); let md5 = format!("{:x}", md5::compute(&bytes)); let revision = Revision::new(&document_id, 0, 1, bytes, &self.user_id, md5); - let record = RevisionRecord::new(revision); + let record = SyncRecord::new(revision); match disk_cache.create_revision_records(vec![record]) { Ok(_) => {} Err(err) => { diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs index 9aa9e2e77b..7a4485861f 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{RevType, Revision, RevisionRange}, util::md5, @@ -23,7 +23,7 @@ pub struct SQLiteDeltaDocumentRevisionPersistence { impl RevisionDiskCache> for SQLiteDeltaDocumentRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = DeltaRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -37,7 +37,7 @@ impl RevisionDiskCache> for SQLiteDeltaDocumentRevisionPersi &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = DeltaRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -47,7 +47,7 @@ impl RevisionDiskCache> for SQLiteDeltaDocumentRevisionPersi &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = DeltaRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -74,7 +74,7 @@ impl RevisionDiskCache> for SQLiteDeltaDocumentRevisionPersi &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -97,7 +97,7 @@ impl SQLiteDeltaDocumentRevisionPersistence { pub struct DeltaRevisionSql {} impl DeltaRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records @@ -143,7 +143,7 @@ impl DeltaRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::rev_table.filter(dsl::doc_id.eq(object_id)).into_boxed(); if let Some(rev_ids) = rev_ids { sql = sql.filter(dsl::rev_id.eq_any(rev_ids)); @@ -162,7 +162,7 @@ impl DeltaRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -244,7 +244,7 @@ impl std::default::Default for TextRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.doc_id, @@ -254,7 +254,7 @@ fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> Revisio user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs index da82a4eee9..d02528a79f 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{Revision, RevisionRange}, util::md5, @@ -22,7 +22,7 @@ pub struct SQLiteDocumentRevisionPersistence { impl RevisionDiskCache> for SQLiteDocumentRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = DocumentRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -36,7 +36,7 @@ impl RevisionDiskCache> for SQLiteDocumentRevisionPersistenc &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = DocumentRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -46,7 +46,7 @@ impl RevisionDiskCache> for SQLiteDocumentRevisionPersistenc &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = DocumentRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -73,7 +73,7 @@ impl RevisionDiskCache> for SQLiteDocumentRevisionPersistenc &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -96,7 +96,7 @@ impl SQLiteDocumentRevisionPersistence { struct DocumentRevisionSql {} impl DocumentRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records .into_iter() @@ -142,7 +142,7 @@ impl DocumentRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::document_rev_table .filter(dsl::document_id.eq(object_id)) .into_boxed(); @@ -163,7 +163,7 @@ impl DocumentRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::document_rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -220,7 +220,7 @@ impl std::default::Default for DocumentRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: DocumentRevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: DocumentRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.document_id, @@ -230,7 +230,7 @@ fn mk_revision_record_from_table(user_id: &str, table: DocumentRevisionTable) -> user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/migration.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/migration.rs index 7f363980ab..49ff81f128 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/migration.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/migration.rs @@ -9,11 +9,12 @@ use flowy_error::{FlowyError, FlowyResult}; use flowy_folder_data_model::revision::{AppRevision, FolderRevision, ViewRevision, WorkspaceRevision}; use flowy_revision::reset::{RevisionResettable, RevisionStructReset}; use flowy_sync::client_folder::make_folder_rev_json_str; +use flowy_sync::client_folder::FolderPad; use flowy_sync::entities::revision::Revision; use flowy_sync::server_folder::FolderOperationsBuilder; -use flowy_sync::{client_folder::FolderPad, entities::revision::md5}; use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence; +use flowy_sync::util::md5; use std::sync::Arc; const V1_MIGRATION: &str = "FOLDER_V1_MIGRATION"; diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs index a0da34aee2..147effff9b 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs @@ -11,7 +11,7 @@ use crate::{ use flowy_database::ConnectionPool; use flowy_error::{FlowyError, FlowyResult}; use flowy_folder_data_model::revision::{AppRevision, TrashRevision, ViewRevision, WorkspaceRevision}; -use flowy_revision::disk::{RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{client_folder::FolderPad, entities::revision::Revision}; use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence; @@ -112,7 +112,7 @@ impl FolderPersistence { let json = folder.to_json()?; let delta_data = FolderOperationsBuilder::new().insert(&json).build().json_bytes(); let revision = Revision::initial_revision(user_id, folder_id.as_ref(), delta_data); - let record = RevisionRecord { + let record = SyncRecord { revision, state: RevisionState::Sync, write_to_disk: true, diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs index a80c95ba15..c094d4be8c 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{RevType, Revision, RevisionRange}, util::md5, @@ -23,7 +23,7 @@ pub struct SQLiteFolderRevisionPersistence { impl RevisionDiskCache> for SQLiteFolderRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = FolderRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -37,7 +37,7 @@ impl RevisionDiskCache> for SQLiteFolderRevisionPersistence &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = FolderRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -47,7 +47,7 @@ impl RevisionDiskCache> for SQLiteFolderRevisionPersistence &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = FolderRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -74,7 +74,7 @@ impl RevisionDiskCache> for SQLiteFolderRevisionPersistence &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -97,7 +97,7 @@ impl SQLiteFolderRevisionPersistence { struct FolderRevisionSql {} impl FolderRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records @@ -143,7 +143,7 @@ impl FolderRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::rev_table.filter(dsl::doc_id.eq(object_id)).into_boxed(); if let Some(rev_ids) = rev_ids { sql = sql.filter(dsl::rev_id.eq_any(rev_ids)); @@ -162,7 +162,7 @@ impl FolderRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -220,7 +220,7 @@ impl std::default::Default for TextRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.doc_id, @@ -230,7 +230,7 @@ fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> Revisio user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-folder/src/services/web_socket.rs b/frontend/rust-lib/flowy-folder/src/services/web_socket.rs index 204ebc7f04..106e7ccb4b 100644 --- a/frontend/rust-lib/flowy-folder/src/services/web_socket.rs +++ b/frontend/rust-lib/flowy-folder/src/services/web_socket.rs @@ -78,12 +78,12 @@ struct FolderConflictResolver { } impl ConflictResolver for FolderConflictResolver { - fn compose_operations(&self, operations: FolderResolveOperations) -> BoxResultFuture { + fn compose_operations(&self, operations: FolderResolveOperations) -> BoxResultFuture { let operations = operations.into_inner(); let folder_pad = self.folder_pad.clone(); Box::pin(async move { let md5 = folder_pad.write().compose_remote_operations(operations)?; - Ok(md5) + Ok(md5.into()) }) } @@ -113,11 +113,11 @@ impl ConflictResolver for FolderConflictResolver { }) } - fn reset_operations(&self, operations: FolderResolveOperations) -> BoxResultFuture { + fn reset_operations(&self, operations: FolderResolveOperations) -> BoxResultFuture { let folder_pad = self.folder_pad.clone(); Box::pin(async move { let md5 = folder_pad.write().reset_folder(operations.into_inner())?; - Ok(md5) + Ok(md5.into()) }) } } diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index 9f91f64c98..72c6e808dc 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -15,7 +15,7 @@ use flowy_error::{FlowyError, FlowyResult}; use flowy_grid_data_model::revision::{BuildGridContext, GridRevision, GridViewRevision}; use flowy_revision::{RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence}; use flowy_sync::client_grid::{make_grid_block_operations, make_grid_operations, make_grid_view_operations}; -use flowy_sync::entities::revision::{RepeatedRevision, Revision}; +use flowy_sync::entities::revision::Revision; use std::sync::Arc; use tokio::sync::RwLock; @@ -67,7 +67,7 @@ impl GridManager { } #[tracing::instrument(level = "debug", skip_all, err)] - pub async fn create_grid>(&self, grid_id: T, revisions: RepeatedRevision) -> FlowyResult<()> { + pub async fn create_grid>(&self, grid_id: T, revisions: Vec) -> FlowyResult<()> { let grid_id = grid_id.as_ref(); let db_pool = self.grid_user.db_pool()?; let rev_manager = self.make_grid_rev_manager(grid_id, db_pool)?; @@ -77,7 +77,7 @@ impl GridManager { } #[tracing::instrument(level = "debug", skip_all, err)] - async fn create_grid_view>(&self, view_id: T, revisions: RepeatedRevision) -> FlowyResult<()> { + async fn create_grid_view>(&self, view_id: T, revisions: Vec) -> FlowyResult<()> { let view_id = view_id.as_ref(); let rev_manager = make_grid_view_rev_manager(&self.grid_user, view_id).await?; let _ = rev_manager.reset_object(revisions).await?; @@ -85,7 +85,7 @@ impl GridManager { } #[tracing::instrument(level = "debug", skip_all, err)] - pub async fn create_grid_block>(&self, block_id: T, revisions: RepeatedRevision) -> FlowyResult<()> { + pub async fn create_grid_block>(&self, block_id: T, revisions: Vec) -> FlowyResult<()> { let block_id = block_id.as_ref(); let db_pool = self.grid_user.db_pool()?; let rev_manager = self.make_grid_block_rev_manager(block_id, db_pool)?; @@ -208,9 +208,8 @@ pub async fn make_grid_view_data( // Create grid's block let grid_block_delta = make_grid_block_operations(block_meta_data); let block_delta_data = grid_block_delta.json_bytes(); - let repeated_revision: RepeatedRevision = - Revision::initial_revision(user_id, block_id, block_delta_data).into(); - let _ = grid_manager.create_grid_block(&block_id, repeated_revision).await?; + let revision = Revision::initial_revision(user_id, block_id, block_delta_data); + let _ = grid_manager.create_grid_block(&block_id, vec![revision]).await?; } // Will replace the grid_id with the value returned by the gen_grid_id() @@ -220,9 +219,8 @@ pub async fn make_grid_view_data( // Create grid let grid_rev_delta = make_grid_operations(&grid_rev); let grid_rev_delta_bytes = grid_rev_delta.json_bytes(); - let repeated_revision: RepeatedRevision = - Revision::initial_revision(user_id, &grid_id, grid_rev_delta_bytes.clone()).into(); - let _ = grid_manager.create_grid(&grid_id, repeated_revision).await?; + let revision = Revision::initial_revision(user_id, &grid_id, grid_rev_delta_bytes.clone()); + let _ = grid_manager.create_grid(&grid_id, vec![revision]).await?; // Create grid view let grid_view = if grid_view_revision_data.is_empty() { @@ -232,9 +230,8 @@ pub async fn make_grid_view_data( }; let grid_view_delta = make_grid_view_operations(&grid_view); let grid_view_delta_bytes = grid_view_delta.json_bytes(); - let repeated_revision: RepeatedRevision = - Revision::initial_revision(user_id, view_id, grid_view_delta_bytes).into(); - let _ = grid_manager.create_grid_view(view_id, repeated_revision).await?; + let revision = Revision::initial_revision(user_id, view_id, grid_view_delta_bytes); + let _ = grid_manager.create_grid_view(view_id, vec![revision]).await?; Ok(grid_rev_delta_bytes) } diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs index 27b4e7790b..a4895ae5e8 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{Revision, RevisionRange}, util::md5, @@ -22,7 +22,7 @@ pub struct SQLiteGridBlockRevisionPersistence { impl RevisionDiskCache> for SQLiteGridBlockRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = GridMetaRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -36,7 +36,7 @@ impl RevisionDiskCache> for SQLiteGridBlockRevisionPersisten &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = GridMetaRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -46,7 +46,7 @@ impl RevisionDiskCache> for SQLiteGridBlockRevisionPersisten &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = GridMetaRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -73,7 +73,7 @@ impl RevisionDiskCache> for SQLiteGridBlockRevisionPersisten &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -95,7 +95,7 @@ impl SQLiteGridBlockRevisionPersistence { struct GridMetaRevisionSql(); impl GridMetaRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records @@ -142,7 +142,7 @@ impl GridMetaRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::grid_meta_rev_table .filter(dsl::object_id.eq(object_id)) .into_boxed(); @@ -163,7 +163,7 @@ impl GridMetaRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::grid_meta_rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -219,7 +219,7 @@ impl std::default::Default for GridBlockRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridBlockRevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: GridBlockRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, @@ -229,7 +229,7 @@ fn mk_revision_record_from_table(user_id: &str, table: GridBlockRevisionTable) - user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs index 5e86286f22..3fd121c5e3 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{Revision, RevisionRange}, util::md5, @@ -22,7 +22,7 @@ pub struct SQLiteGridRevisionPersistence { impl RevisionDiskCache> for SQLiteGridRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = GridRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -36,7 +36,7 @@ impl RevisionDiskCache> for SQLiteGridRevisionPersistence { &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = GridRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -46,7 +46,7 @@ impl RevisionDiskCache> for SQLiteGridRevisionPersistence { &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = GridRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -73,7 +73,7 @@ impl RevisionDiskCache> for SQLiteGridRevisionPersistence { &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -95,7 +95,7 @@ impl SQLiteGridRevisionPersistence { struct GridRevisionSql(); impl GridRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records .into_iter() @@ -141,7 +141,7 @@ impl GridRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::grid_rev_table.filter(dsl::object_id.eq(object_id)).into_boxed(); if let Some(rev_ids) = rev_ids { sql = sql.filter(dsl::rev_id.eq_any(rev_ids)); @@ -160,7 +160,7 @@ impl GridRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::grid_rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -217,7 +217,7 @@ impl std::default::Default for GridRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridRevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: GridRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, @@ -227,7 +227,7 @@ fn mk_revision_record_from_table(user_id: &str, table: GridRevisionTable) -> Rev user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs index 737e7eaece..fe9490d0de 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs @@ -7,7 +7,7 @@ use flowy_database::{ ConnectionPool, }; use flowy_error::{internal_error, FlowyError, FlowyResult}; -use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ entities::revision::{Revision, RevisionRange}, util::md5, @@ -31,7 +31,7 @@ impl SQLiteGridViewRevisionPersistence { impl RevisionDiskCache> for SQLiteGridViewRevisionPersistence { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let _ = GridViewRevisionSql::create(revision_records, &*conn)?; Ok(()) @@ -45,7 +45,7 @@ impl RevisionDiskCache> for SQLiteGridViewRevisionPersistenc &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = self.pool.get().map_err(internal_error)?; let records = GridViewRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?; Ok(records) @@ -55,7 +55,7 @@ impl RevisionDiskCache> for SQLiteGridViewRevisionPersistenc &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { let conn = &*self.pool.get().map_err(internal_error)?; let revisions = GridViewRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?; Ok(revisions) @@ -82,7 +82,7 @@ impl RevisionDiskCache> for SQLiteGridViewRevisionPersistenc &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { let conn = self.pool.get().map_err(internal_error)?; conn.immediate_transaction::<_, FlowyError, _>(|| { @@ -95,7 +95,7 @@ impl RevisionDiskCache> for SQLiteGridViewRevisionPersistenc struct GridViewRevisionSql(); impl GridViewRevisionSql { - fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { + fn create(revision_records: Vec, conn: &SqliteConnection) -> Result<(), FlowyError> { // Batch insert: https://diesel.rs/guides/all-about-inserts.html let records = revision_records .into_iter() @@ -141,7 +141,7 @@ impl GridViewRevisionSql { object_id: &str, rev_ids: Option>, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let mut sql = dsl::grid_view_rev_table .filter(dsl::object_id.eq(object_id)) .into_boxed(); @@ -162,7 +162,7 @@ impl GridViewRevisionSql { object_id: &str, range: RevisionRange, conn: &SqliteConnection, - ) -> Result, FlowyError> { + ) -> Result, FlowyError> { let rev_tables = dsl::grid_view_rev_table .filter(dsl::rev_id.ge(range.start)) .filter(dsl::rev_id.le(range.end)) @@ -219,7 +219,7 @@ impl std::default::Default for GridViewRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridViewRevisionTable) -> RevisionRecord { +fn mk_revision_record_from_table(user_id: &str, table: GridViewRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, @@ -229,7 +229,7 @@ fn mk_revision_record_from_table(user_id: &str, table: GridViewRevisionTable) -> user_id, md5, ); - RevisionRecord { + SyncRecord { revision, state: table.state.into(), write_to_disk: false, diff --git a/frontend/rust-lib/flowy-revision/src/cache/disk.rs b/frontend/rust-lib/flowy-revision/src/cache/disk.rs index 06e85f9d5e..f89f36367b 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/disk.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/disk.rs @@ -5,23 +5,20 @@ use std::sync::Arc; pub trait RevisionDiskCache: Sync + Send { type Error: Debug; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error>; + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error>; fn get_connection(&self) -> Result; // Read all the records if the rev_ids is None - fn read_revision_records( - &self, - object_id: &str, - rev_ids: Option>, - ) -> Result, Self::Error>; + fn read_revision_records(&self, object_id: &str, rev_ids: Option>) + -> Result, Self::Error>; // Read the revision which rev_id >= range.start && rev_id <= range.end fn read_revision_records_with_range( &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error>; + ) -> Result, Self::Error>; fn update_revision_record(&self, changesets: Vec) -> FlowyResult<()>; @@ -34,7 +31,7 @@ pub trait RevisionDiskCache: Sync + Send { &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error>; } @@ -44,7 +41,7 @@ where { type Error = FlowyError; - fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { (**self).create_revision_records(revision_records) } @@ -56,7 +53,7 @@ where &self, object_id: &str, rev_ids: Option>, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { (**self).read_revision_records(object_id, rev_ids) } @@ -64,7 +61,7 @@ where &self, object_id: &str, range: &RevisionRange, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { (**self).read_revision_records_with_range(object_id, range) } @@ -80,20 +77,20 @@ where &self, object_id: &str, deleted_rev_ids: Option>, - inserted_records: Vec, + inserted_records: Vec, ) -> Result<(), Self::Error> { (**self).delete_and_insert_records(object_id, deleted_rev_ids, inserted_records) } } #[derive(Clone, Debug)] -pub struct RevisionRecord { +pub struct SyncRecord { pub revision: Revision, pub state: RevisionState, pub write_to_disk: bool, } -impl RevisionRecord { +impl SyncRecord { pub fn new(revision: Revision) -> Self { Self { revision, diff --git a/frontend/rust-lib/flowy-revision/src/cache/memory.rs b/frontend/rust-lib/flowy-revision/src/cache/memory.rs index 6120c3f224..ad6795437c 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/memory.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/memory.rs @@ -1,4 +1,4 @@ -use crate::disk::RevisionRecord; +use crate::disk::SyncRecord; use crate::REVISION_WRITE_INTERVAL_IN_MILLIS; use dashmap::DashMap; use flowy_error::{FlowyError, FlowyResult}; @@ -7,15 +7,15 @@ use std::{borrow::Cow, sync::Arc, time::Duration}; use tokio::{sync::RwLock, task::JoinHandle}; pub(crate) trait RevisionMemoryCacheDelegate: Send + Sync { - fn checkpoint_tick(&self, records: Vec) -> FlowyResult<()>; + fn send_sync(&self, records: Vec) -> FlowyResult<()>; fn receive_ack(&self, object_id: &str, rev_id: i64); } pub(crate) struct RevisionMemoryCache { object_id: String, - revs_map: Arc>, + revs_map: Arc>, delegate: Arc, - pending_write_revs: Arc>>, + defer_write_revs: Arc>>, defer_save: RwLock>>, } @@ -25,7 +25,7 @@ impl RevisionMemoryCache { object_id: object_id.to_owned(), revs_map: Arc::new(DashMap::new()), delegate, - pending_write_revs: Arc::new(RwLock::new(vec![])), + defer_write_revs: Arc::new(RwLock::new(vec![])), defer_save: RwLock::new(None), } } @@ -34,7 +34,7 @@ impl RevisionMemoryCache { self.revs_map.contains_key(rev_id) } - pub(crate) async fn add<'a>(&'a self, record: Cow<'a, RevisionRecord>) { + pub(crate) async fn add<'a>(&'a self, record: Cow<'a, SyncRecord>) { let record = match record { Cow::Borrowed(record) => record.clone(), Cow::Owned(record) => record, @@ -43,11 +43,11 @@ impl RevisionMemoryCache { let rev_id = record.revision.rev_id; self.revs_map.insert(rev_id, record); - let mut write_guard = self.pending_write_revs.write().await; + let mut write_guard = self.defer_write_revs.write().await; if !write_guard.contains(&rev_id) { write_guard.push(rev_id); drop(write_guard); - self.make_checkpoint().await; + self.tick_checkpoint().await; } } @@ -57,8 +57,8 @@ impl RevisionMemoryCache { Some(mut record) => record.ack(), } - if self.pending_write_revs.read().await.contains(rev_id) { - self.make_checkpoint().await; + if self.defer_write_revs.read().await.contains(rev_id) { + self.tick_checkpoint().await; } else { // The revision must be saved on disk if the pending_write_revs // doesn't contains the rev_id. @@ -66,7 +66,7 @@ impl RevisionMemoryCache { } } - pub(crate) async fn get(&self, rev_id: &i64) -> Option { + pub(crate) async fn get(&self, rev_id: &i64) -> Option { self.revs_map.get(rev_id).map(|r| r.value().clone()) } @@ -80,21 +80,21 @@ impl RevisionMemoryCache { } } - pub(crate) async fn get_with_range(&self, range: &RevisionRange) -> Result, FlowyError> { + pub(crate) async fn get_with_range(&self, range: &RevisionRange) -> Result, FlowyError> { let revs = range .iter() .flat_map(|rev_id| self.revs_map.get(&rev_id).map(|record| record.clone())) - .collect::>(); + .collect::>(); Ok(revs) } - pub(crate) async fn reset_with_revisions(&self, revision_records: Vec) { + pub(crate) async fn reset_with_revisions(&self, revision_records: Vec) { self.revs_map.clear(); if let Some(handler) = self.defer_save.write().await.take() { handler.abort(); } - let mut write_guard = self.pending_write_revs.write().await; + let mut write_guard = self.defer_write_revs.write().await; write_guard.clear(); for record in revision_records { write_guard.push(record.revision.rev_id); @@ -102,21 +102,21 @@ impl RevisionMemoryCache { } drop(write_guard); - self.make_checkpoint().await; + self.tick_checkpoint().await; } - async fn make_checkpoint(&self) { + async fn tick_checkpoint(&self) { // https://github.com/async-graphql/async-graphql/blob/ed8449beec3d9c54b94da39bab33cec809903953/src/dataloader/mod.rs#L362 if let Some(handler) = self.defer_save.write().await.take() { handler.abort(); } - if self.pending_write_revs.read().await.is_empty() { + if self.defer_write_revs.read().await.is_empty() { return; } let rev_map = self.revs_map.clone(); - let pending_write_revs = self.pending_write_revs.clone(); + let pending_write_revs = self.defer_write_revs.clone(); let delegate = self.delegate.clone(); *self.defer_save.write().await = Some(tokio::spawn(async move { @@ -128,7 +128,7 @@ impl RevisionMemoryCache { // // Use saturating_sub and split_off ? // https://stackoverflow.com/questions/28952411/what-is-the-idiomatic-way-to-pop-the-last-n-elements-in-a-mutable-vec - let mut save_records: Vec = vec![]; + let mut save_records: Vec = vec![]; revs_write_guard.iter().for_each(|rev_id| match rev_map.get(rev_id) { None => {} Some(value) => { @@ -136,7 +136,7 @@ impl RevisionMemoryCache { } }); - if delegate.checkpoint_tick(save_records).is_ok() { + if delegate.send_sync(save_records).is_ok() { revs_write_guard.clear(); drop(revs_write_guard); } diff --git a/frontend/rust-lib/flowy-revision/src/cache/reset.rs b/frontend/rust-lib/flowy-revision/src/cache/reset.rs index bd128a9d9e..a1cb58a6bd 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/reset.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/reset.rs @@ -1,4 +1,4 @@ -use crate::disk::{RevisionDiskCache, RevisionRecord}; +use crate::disk::{RevisionDiskCache, SyncRecord}; use crate::{RevisionLoader, RevisionPersistence}; use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; @@ -76,7 +76,7 @@ where let bytes = self.target.reset_data(revisions)?; let revision = Revision::initial_revision(&self.user_id, self.target.target_id(), bytes); - let record = RevisionRecord::new(revision); + let record = SyncRecord::new(revision); tracing::trace!("Reset {} revision record object", self.target.target_id()); let _ = self diff --git a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs index e48cb59407..4fb8260311 100644 --- a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs +++ b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs @@ -1,4 +1,4 @@ -use crate::RevisionManager; +use crate::{RevisionMD5, RevisionManager}; use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_sync::entities::{ @@ -8,8 +8,6 @@ use flowy_sync::entities::{ use lib_infra::future::BoxResultFuture; use std::{convert::TryFrom, sync::Arc}; -pub type OperationsMD5 = String; - pub struct TransformOperations { pub client_operations: Operations, pub server_operations: Option, @@ -28,12 +26,12 @@ pub trait ConflictResolver where Operations: Send + Sync, { - fn compose_operations(&self, operations: Operations) -> BoxResultFuture; + fn compose_operations(&self, operations: Operations) -> BoxResultFuture; fn transform_operations( &self, operations: Operations, ) -> BoxResultFuture, FlowyError>; - fn reset_operations(&self, operations: Operations) -> BoxResultFuture; + fn reset_operations(&self, operations: Operations) -> BoxResultFuture; } pub trait ConflictRevisionSink: Send + Sync + 'static { @@ -129,9 +127,8 @@ where // The server_prime is None means the client local revisions conflict with the // // server, and it needs to override the client delta. let md5 = self.resolver.reset_operations(client_operations).await?; - let repeated_revision = RepeatedRevision::new(revisions); - assert_eq!(repeated_revision.last().unwrap().md5, md5); - let _ = self.rev_manager.reset_object(repeated_revision).await?; + debug_assert!(md5.is_equal(&revisions.last().unwrap().md5)); + let _ = self.rev_manager.reset_object(revisions).await?; Ok(None) } Some(server_operations) => { @@ -158,7 +155,7 @@ fn make_client_and_server_revision( rev_manager: &Arc>, client_operations: Operations, server_operations: Option, - md5: String, + md5: RevisionMD5, ) -> (Revision, Option) where Operations: OperationsSerializer, diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index 0b89de828f..49348ccc98 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -3,8 +3,8 @@ use crate::{RevisionPersistence, RevisionSnapshotDiskCache, RevisionSnapshotMana use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_sync::{ - entities::revision::{RepeatedRevision, Revision, RevisionRange}, - util::{pair_rev_id_from_revisions, RevIdCounter}, + entities::revision::{Revision, RevisionRange}, + util::{md5, pair_rev_id_from_revisions, RevIdCounter}, }; use lib_infra::future::FutureResult; use std::sync::Arc; @@ -143,9 +143,9 @@ impl RevisionManager { } #[tracing::instrument(level = "debug", skip(self, revisions), err)] - pub async fn reset_object(&self, revisions: RepeatedRevision) -> FlowyResult<()> { + pub async fn reset_object(&self, revisions: Vec) -> FlowyResult<()> { let rev_id = pair_rev_id_from_revisions(&revisions).1; - let _ = self.rev_persistence.reset(revisions.into_inner()).await?; + let _ = self.rev_persistence.reset(revisions).await?; self.rev_id_counter.set(rev_id); Ok(()) } @@ -191,7 +191,7 @@ impl RevisionManager { pub fn next_rev_id_pair(&self) -> (i64, i64) { let cur = self.rev_id_counter.value(); - let next = self.rev_id_counter.next(); + let next = self.rev_id_counter.next_id(); (cur, next) } @@ -283,3 +283,56 @@ impl RevisionLoader { Ok(revisions) } } + +/// Represents as the md5 of the revision object after applying the +/// revision. For example, RevisionMD5 will be the md5 of the document +/// content. +#[derive(Debug, Clone)] +pub struct RevisionMD5(String); + +impl RevisionMD5 { + pub fn from_bytes>(bytes: T) -> Result { + Ok(RevisionMD5(md5(bytes))) + } + + pub fn into_inner(self) -> String { + self.0 + } + + pub fn is_equal(&self, s: &str) -> bool { + self.0 == s + } +} + +impl std::convert::From for String { + fn from(md5: RevisionMD5) -> Self { + md5.0 + } +} + +impl std::convert::From<&str> for RevisionMD5 { + fn from(s: &str) -> Self { + Self(s.to_owned()) + } +} +impl std::convert::From for RevisionMD5 { + fn from(s: String) -> Self { + Self(s) + } +} + +impl std::ops::Deref for RevisionMD5 { + type Target = String; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for RevisionMD5 { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl std::cmp::Eq for RevisionMD5 {} diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index 7a58cee1d8..e0c9529c1f 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -2,10 +2,9 @@ use crate::cache::{ disk::{RevisionChangeset, RevisionDiskCache}, memory::RevisionMemoryCacheDelegate, }; -use crate::disk::{RevisionRecord, RevisionState}; +use crate::disk::{RevisionState, SyncRecord}; use crate::memory::RevisionMemoryCache; use crate::RevisionCompress; - use flowy_error::{internal_error, FlowyError, FlowyResult}; use flowy_sync::entities::revision::{Revision, RevisionRange}; use std::collections::VecDeque; @@ -20,10 +19,13 @@ pub struct RevisionPersistence { object_id: String, disk_cache: Arc>, memory_cache: Arc, - sync_seq: RwLock, + sync_seq: RwLock, } -impl RevisionPersistence { +impl RevisionPersistence +where + Connection: 'static, +{ pub fn new(user_id: &str, object_id: &str, disk_cache: C) -> RevisionPersistence where C: 'static + RevisionDiskCache, @@ -39,7 +41,7 @@ impl RevisionPersistence { ) -> RevisionPersistence { let object_id = object_id.to_owned(); let user_id = user_id.to_owned(); - let sync_seq = RwLock::new(RevisionSyncSequence::new()); + let sync_seq = RwLock::new(DeferSyncSequence::new()); let memory_cache = Arc::new(RevisionMemoryCache::new(&object_id, Arc::new(disk_cache.clone()))); Self { user_id, @@ -131,7 +133,7 @@ impl RevisionPersistence { pub(crate) async fn reset(&self, revisions: Vec) -> FlowyResult<()> { let records = revisions .into_iter() - .map(|revision| RevisionRecord { + .map(|revision| SyncRecord { revision, state: RevisionState::Sync, write_to_disk: false, @@ -151,7 +153,7 @@ impl RevisionPersistence { tracing::warn!("Duplicate revision: {}:{}-{:?}", self.object_id, revision.rev_id, state); return Ok(()); } - let record = RevisionRecord { + let record = SyncRecord { revision, state, write_to_disk, @@ -172,7 +174,7 @@ impl RevisionPersistence { Ok(()) } - pub async fn get(&self, rev_id: i64) -> Option { + pub async fn get(&self, rev_id: i64) -> Option { match self.memory_cache.get(&rev_id).await { None => match self .disk_cache @@ -192,7 +194,7 @@ impl RevisionPersistence { } } - pub fn batch_get(&self, doc_id: &str) -> FlowyResult> { + pub fn batch_get(&self, doc_id: &str) -> FlowyResult> { self.disk_cache.read_revision_records(doc_id, None) } @@ -225,7 +227,7 @@ impl RevisionPersistence { } impl RevisionMemoryCacheDelegate for Arc> { - fn checkpoint_tick(&self, mut records: Vec) -> FlowyResult<()> { + fn send_sync(&self, mut records: Vec) -> FlowyResult<()> { records.retain(|record| record.write_to_disk); if !records.is_empty() { tracing::Span::current().record( @@ -251,10 +253,10 @@ impl RevisionMemoryCacheDelegate for Arc); -impl RevisionSyncSequence { +struct DeferSyncSequence(VecDeque); +impl DeferSyncSequence { fn new() -> Self { - RevisionSyncSequence::default() + DeferSyncSequence::default() } fn add(&mut self, new_rev_id: i64) -> FlowyResult<()> { diff --git a/frontend/rust-lib/flowy-revision/src/ws_manager.rs b/frontend/rust-lib/flowy-revision/src/ws_manager.rs index eb7539c380..7413cf37d0 100644 --- a/frontend/rust-lib/flowy-revision/src/ws_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/ws_manager.rs @@ -28,7 +28,7 @@ pub trait RevisionWSDataStream: Send + Sync { } // The sink provides the data that will be sent through the web socket to the -// backend. +// server. pub trait RevisionWebSocketSink: Send + Sync { fn next(&self) -> FutureResult, FlowyError>; } diff --git a/frontend/rust-lib/flowy-revision/tests/main.rs b/frontend/rust-lib/flowy-revision/tests/main.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/main.rs @@ -0,0 +1 @@ + diff --git a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs index 4625bfa6bf..0320d078e9 100644 --- a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs +++ b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs @@ -18,7 +18,7 @@ use flowy_net::{ http_server::folder::FolderHttpCloudService, local_server::LocalServer, ws::connection::FlowyWebSocketConnect, }; use flowy_revision::{RevisionWebSocket, WSStateReceiver}; -use flowy_sync::entities::revision::{RepeatedRevision, Revision}; +use flowy_sync::entities::revision::Revision; use flowy_sync::entities::ws_data::ClientRevisionWSData; use flowy_user::services::UserSession; use futures_core::future::BoxFuture; @@ -151,12 +151,12 @@ impl ViewDataProcessor for DocumentViewDataProcessor { ) -> FutureResult<(), FlowyError> { // Only accept Document type debug_assert_eq!(layout, ViewLayoutTypePB::Document); - let repeated_revision: RepeatedRevision = Revision::initial_revision(user_id, view_id, view_data).into(); + let revision = Revision::initial_revision(user_id, view_id, view_data); let view_id = view_id.to_string(); let manager = self.0.clone(); FutureResult::new(async move { - let _ = manager.create_document(view_id, repeated_revision).await?; + let _ = manager.create_document(view_id, vec![revision]).await?; Ok(()) }) } @@ -194,9 +194,8 @@ impl ViewDataProcessor for DocumentViewDataProcessor { let document_content = self.0.initial_document_content(); FutureResult::new(async move { let delta_data = Bytes::from(document_content); - let repeated_revision: RepeatedRevision = - Revision::initial_revision(&user_id, &view_id, delta_data.clone()).into(); - let _ = manager.create_document(view_id, repeated_revision).await?; + let revision = Revision::initial_revision(&user_id, &view_id, delta_data.clone()); + let _ = manager.create_document(view_id, vec![revision]).await?; Ok(delta_data) }) } @@ -226,11 +225,11 @@ impl ViewDataProcessor for GridViewDataProcessor { _layout: ViewLayoutTypePB, delta_data: Bytes, ) -> FutureResult<(), FlowyError> { - let repeated_revision: RepeatedRevision = Revision::initial_revision(user_id, view_id, delta_data).into(); + let revision = Revision::initial_revision(user_id, view_id, delta_data); let view_id = view_id.to_string(); let grid_manager = self.0.clone(); FutureResult::new(async move { - let _ = grid_manager.create_grid(view_id, repeated_revision).await?; + let _ = grid_manager.create_grid(view_id, vec![revision]).await?; Ok(()) }) } diff --git a/shared-lib/flowy-sync/src/client_document/document_pad.rs b/shared-lib/flowy-sync/src/client_document/document_pad.rs index be438dca25..ab8863c69b 100644 --- a/shared-lib/flowy-sync/src/client_document/document_pad.rs +++ b/shared-lib/flowy-sync/src/client_document/document_pad.rs @@ -1,3 +1,4 @@ +use crate::util::md5; use crate::{ client_document::{ history::{History, UndoResult}, @@ -77,9 +78,9 @@ impl ClientDocument { &self.operations } - pub fn md5(&self) -> String { + pub fn document_md5(&self) -> String { let bytes = self.to_bytes(); - format!("{:x}", md5::compute(bytes)) + md5(&bytes) } pub fn set_notify(&mut self, notify: mpsc::UnboundedSender<()>) { diff --git a/shared-lib/flowy-sync/src/client_folder/folder_pad.rs b/shared-lib/flowy-sync/src/client_folder/folder_pad.rs index e11d73963d..bb1dfb6902 100644 --- a/shared-lib/flowy-sync/src/client_folder/folder_pad.rs +++ b/shared-lib/flowy-sync/src/client_folder/folder_pad.rs @@ -1,9 +1,9 @@ use crate::errors::internal_error; use crate::server_folder::{FolderOperations, FolderOperationsBuilder}; -use crate::util::cal_diff; +use crate::util::{cal_diff, md5}; use crate::{ client_folder::builder::FolderPadBuilder, - entities::revision::{md5, Revision}, + entities::revision::Revision, errors::{CollaborateError, CollaborateResult}, }; use flowy_folder_data_model::revision::{AppRevision, FolderRevision, TrashRevision, ViewRevision, WorkspaceRevision}; @@ -61,7 +61,7 @@ impl FolderPad { self.folder_rev = folder.folder_rev; self.operations = folder.operations; - Ok(self.md5()) + Ok(self.folder_md5()) } pub fn compose_remote_operations(&mut self, operations: FolderOperations) -> CollaborateResult { @@ -313,7 +313,7 @@ impl FolderPad { } } - pub fn md5(&self) -> String { + pub fn folder_md5(&self) -> String { md5(&self.operations.json_bytes()) } @@ -345,7 +345,7 @@ impl FolderPad { self.operations = self.operations.compose(&operations)?; Ok(Some(FolderChangeset { operations, - md5: self.md5(), + md5: self.folder_md5(), })) } } @@ -383,7 +383,7 @@ impl FolderPad { self.operations = self.operations.compose(&operations)?; Ok(Some(FolderChangeset { operations, - md5: self.md5(), + md5: self.folder_md5(), })) } } diff --git a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs index 36f65837c0..c5a504c358 100644 --- a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs @@ -1,6 +1,6 @@ -use crate::entities::revision::{md5, RepeatedRevision, Revision}; +use crate::entities::revision::{RepeatedRevision, Revision}; use crate::errors::{CollaborateError, CollaborateResult}; -use crate::util::{cal_diff, make_operations_from_revisions}; +use crate::util::{cal_diff, make_operations_from_revisions, md5}; use flowy_grid_data_model::revision::{ gen_block_id, gen_row_id, CellRevision, GridBlockRevision, RowChangeset, RowRevision, }; diff --git a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs index f27c46d250..6f99a23e53 100644 --- a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs @@ -1,6 +1,6 @@ -use crate::entities::revision::{md5, RepeatedRevision, Revision}; +use crate::entities::revision::{RepeatedRevision, Revision}; use crate::errors::{internal_error, CollaborateError, CollaborateResult}; -use crate::util::{cal_diff, make_operations_from_revisions}; +use crate::util::{cal_diff, make_operations_from_revisions, md5}; use flowy_grid_data_model::revision::{ gen_block_id, gen_grid_id, FieldRevision, FieldTypeRevision, GridBlockMetaRevision, GridBlockMetaRevisionChangeset, @@ -315,7 +315,7 @@ impl GridRevisionPad { }) } - pub fn md5(&self) -> String { + pub fn grid_md5(&self) -> String { md5(&self.operations.json_bytes()) } @@ -343,7 +343,7 @@ impl GridRevisionPad { self.operations = self.operations.compose(&operations)?; Ok(Some(GridRevisionChangeset { operations, - md5: self.md5(), + md5: self.grid_md5(), })) } } diff --git a/shared-lib/flowy-sync/src/client_grid/view_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/view_revision_pad.rs index 46e91734b4..04dd82d8ea 100644 --- a/shared-lib/flowy-sync/src/client_grid/view_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/view_revision_pad.rs @@ -1,6 +1,6 @@ -use crate::entities::revision::{md5, Revision}; +use crate::entities::revision::Revision; use crate::errors::{internal_error, CollaborateError, CollaborateResult}; -use crate::util::{cal_diff, make_operations_from_revisions}; +use crate::util::{cal_diff, make_operations_from_revisions, md5}; use flowy_grid_data_model::revision::{ FieldRevision, FieldTypeRevision, FilterConfigurationRevision, FilterConfigurationsByFieldId, GridViewRevision, GroupConfigurationRevision, GroupConfigurationsByFieldId, LayoutRevision, diff --git a/shared-lib/flowy-sync/src/entities/revision.rs b/shared-lib/flowy-sync/src/entities/revision.rs index 39fb71a2e6..950e8956db 100644 --- a/shared-lib/flowy-sync/src/entities/revision.rs +++ b/shared-lib/flowy-sync/src/entities/revision.rs @@ -1,3 +1,4 @@ +use crate::util::md5; use bytes::Bytes; use flowy_derive::{ProtoBuf, ProtoBuf_Enum}; use std::{convert::TryFrom, fmt::Formatter, ops::RangeInclusive}; @@ -36,6 +37,34 @@ impl std::convert::From> for Revision { } impl Revision { + pub fn new>( + object_id: &str, + base_rev_id: i64, + rev_id: i64, + bytes: Bytes, + user_id: &str, + md5: T, + ) -> Revision { + let user_id = user_id.to_owned(); + let object_id = object_id.to_owned(); + let bytes = bytes.to_vec(); + let base_rev_id = base_rev_id; + let rev_id = rev_id; + + if base_rev_id != 0 { + debug_assert!(base_rev_id != rev_id); + } + + Self { + base_rev_id, + rev_id, + bytes, + md5: md5.into(), + object_id, + ty: RevType::DeprecatedLocal, + user_id, + } + } pub fn is_empty(&self) -> bool { self.base_rev_id == self.rev_id } @@ -52,28 +81,6 @@ impl Revision { let md5 = md5(&bytes); Self::new(object_id, 0, 0, bytes, user_id, md5) } - - pub fn new(object_id: &str, base_rev_id: i64, rev_id: i64, bytes: Bytes, user_id: &str, md5: String) -> Revision { - let user_id = user_id.to_owned(); - let object_id = object_id.to_owned(); - let bytes = bytes.to_vec(); - let base_rev_id = base_rev_id; - let rev_id = rev_id; - - if base_rev_id != 0 { - debug_assert!(base_rev_id != rev_id); - } - - Self { - base_rev_id, - rev_id, - bytes, - md5, - object_id, - ty: RevType::DeprecatedLocal, - user_id, - } - } } impl std::fmt::Debug for Revision { @@ -209,12 +216,6 @@ impl RevisionRange { } } -#[inline] -pub fn md5>(data: T) -> String { - let md5 = format!("{:x}", md5::compute(data)); - md5 -} - #[derive(Debug, ProtoBuf_Enum, Clone, Eq, PartialEq)] pub enum RevType { DeprecatedLocal = 0, diff --git a/shared-lib/flowy-sync/src/util.rs b/shared-lib/flowy-sync/src/util.rs index 10e977919f..31c4147e60 100644 --- a/shared-lib/flowy-sync/src/util.rs +++ b/shared-lib/flowy-sync/src/util.rs @@ -49,7 +49,8 @@ impl RevIdCounter { pub fn new(n: i64) -> Self { Self(AtomicI64::new(n)) } - pub fn next(&self) -> i64 { + + pub fn next_id(&self) -> i64 { let _ = self.0.fetch_add(1, SeqCst); self.value() } diff --git a/shared-lib/lib-ot/src/core/node_tree/tree.rs b/shared-lib/lib-ot/src/core/node_tree/tree.rs index dfd0a5f77f..16399211ea 100644 --- a/shared-lib/lib-ot/src/core/node_tree/tree.rs +++ b/shared-lib/lib-ot/src/core/node_tree/tree.rs @@ -35,9 +35,19 @@ impl NodeTree { Ok(tree) } - pub fn from_bytes(bytes: Vec, context: NodeTreeContext) -> Result { - let operations = NodeOperations::from_bytes(bytes)?; - Self::from_operations(operations, context) + pub fn from_bytes(bytes: &[u8]) -> Result { + let tree: NodeTree = serde_json::from_slice(bytes).map_err(|e| OTError::serde().context(e))?; + Ok(tree) + } + + pub fn to_bytes(&self) -> Vec { + match serde_json::to_vec(self) { + Ok(bytes) => bytes, + Err(e) => { + tracing::error!("{}", e); + vec![] + } + } } pub fn from_operations>(operations: T, context: NodeTreeContext) -> Result { diff --git a/shared-lib/lib-ot/tests/node/serde_test.rs b/shared-lib/lib-ot/tests/node/serde_test.rs index 25e086c160..6851cd4164 100644 --- a/shared-lib/lib-ot/tests/node/serde_test.rs +++ b/shared-lib/lib-ot/tests/node/serde_test.rs @@ -1,4 +1,6 @@ -use lib_ot::core::{AttributeBuilder, Changeset, NodeData, NodeDataBuilder, NodeOperation, NodeTree, Path}; +use lib_ot::core::{ + AttributeBuilder, Changeset, NodeData, NodeDataBuilder, NodeOperation, NodeTree, NodeTreeContext, Path, +}; use lib_ot::text_delta::DeltaTextOperationBuilder; #[test] @@ -26,6 +28,7 @@ fn operation_insert_node_with_children_serde_test() { r#"{"op":"insert","path":[0,1],"nodes":[{"type":"text","children":[{"type":"sub_text"}]}]}"# ); } + #[test] fn operation_update_node_attributes_serde_test() { let operation = NodeOperation::Update { @@ -102,6 +105,14 @@ fn node_tree_serialize_test() { assert_eq!(json, TREE_JSON); } +#[test] +fn node_tree_serde_test() { + let tree: NodeTree = serde_json::from_str(TREE_JSON).unwrap(); + let bytes = tree.to_bytes(); + let tree = NodeTree::from_bytes(&bytes).unwrap(); + assert_eq!(bytes, tree.to_bytes()); +} + #[allow(dead_code)] const TREE_JSON: &str = r#"{ "type": "editor", From f5dc9ed97564010c6687ae1b49ad16e6aa3c5a5d Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 2 Nov 2022 10:23:54 +0800 Subject: [PATCH 2/9] test: add revision tests --- frontend/rust-lib/Cargo.lock | 1 + frontend/rust-lib/flowy-revision/Cargo.toml | 3 + .../flowy-revision/src/cache/reset.rs | 2 +- .../flowy-revision/src/rev_manager.rs | 1 + .../rust-lib/flowy-revision/tests/main.rs | 2 +- .../flowy-revision/tests/revision_test/mod.rs | 2 + .../revision_test/revision_order_test.rs | 8 + .../tests/revision_test/script.rs | 139 ++++++++++++++++++ .../flowy-sync/src/entities/revision.rs | 16 +- shared-lib/lib-ot/tests/node/serde_test.rs | 4 +- 10 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs create mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs create mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/script.rs diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 3b60ecdefd..af9a252f57 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -1076,6 +1076,7 @@ dependencies = [ "futures-util", "lib-infra", "lib-ws", + "nanoid", "serde", "serde_json", "strum", diff --git a/frontend/rust-lib/flowy-revision/Cargo.toml b/frontend/rust-lib/flowy-revision/Cargo.toml index 8bd37bf946..2599e13a75 100644 --- a/frontend/rust-lib/flowy-revision/Cargo.toml +++ b/frontend/rust-lib/flowy-revision/Cargo.toml @@ -21,5 +21,8 @@ futures-util = "0.3.15" async-stream = "0.3.2" serde_json = {version = "1.0"} +[dev-dependencies] +nanoid = "0.4.0" + [features] flowy_unit_test = [] \ No newline at end of file diff --git a/frontend/rust-lib/flowy-revision/src/cache/reset.rs b/frontend/rust-lib/flowy-revision/src/cache/reset.rs index a1cb58a6bd..6e6872329b 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/reset.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/reset.rs @@ -47,7 +47,7 @@ where let _ = self.save_migrate_record()?; } Some(s) => { - let mut record = MigrationObjectRecord::from_str(&s)?; + let mut record = MigrationObjectRecord::from_str(&s).map_err(|e| FlowyError::serde().context(e))?; let rev_str = self.target.default_target_rev_str()?; if record.len < rev_str.len() { let _ = self.reset_object().await?; diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index 49348ccc98..e8bab34233 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -185,6 +185,7 @@ impl RevisionManager { Ok(()) } + /// Returns the current revision id pub fn rev_id(&self) -> i64 { self.rev_id_counter.value() } diff --git a/frontend/rust-lib/flowy-revision/tests/main.rs b/frontend/rust-lib/flowy-revision/tests/main.rs index 8b13789179..3eb8b414b2 100644 --- a/frontend/rust-lib/flowy-revision/tests/main.rs +++ b/frontend/rust-lib/flowy-revision/tests/main.rs @@ -1 +1 @@ - +mod revision_test; diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs new file mode 100644 index 0000000000..779735a23d --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs @@ -0,0 +1,2 @@ +mod revision_order_test; +mod script; diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs new file mode 100644 index 0000000000..a7970b875c --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs @@ -0,0 +1,8 @@ +use crate::revision_test::script::{RevisionScript::*, RevisionTest}; + +#[tokio::test] +async fn test() { + let test = RevisionTest::new().await; + let scripts = vec![]; + test.run_scripts(scripts).await; +} diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs new file mode 100644 index 0000000000..f2dcc4846a --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -0,0 +1,139 @@ +use bytes::Bytes; +use flowy_error::{FlowyError, FlowyResult}; +use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, SyncRecord}; +use flowy_revision::{ + RevisionCompress, RevisionManager, RevisionPersistence, RevisionSnapshotDiskCache, RevisionSnapshotInfo, +}; +use flowy_sync::entities::revision::{Revision, RevisionRange}; +use nanoid::nanoid; +use std::sync::Arc; + +pub enum RevisionScript { + AddLocalRevision(Revision), + AckRevision { rev_id: i64 }, + AssertNextSyncRevisionId { rev_id: i64 }, + AssertNextSyncRevision(Option), +} + +pub struct RevisionTest { + rev_manager: Arc>, +} + +impl RevisionTest { + pub async fn new() -> Self { + let user_id = nanoid!(10); + let object_id = nanoid!(6); + let persistence = RevisionPersistence::new(&user_id, &object_id, RevisionDiskCacheMock::new()); + let compress = RevisionCompressMock {}; + let snapshot = RevisionSnapshotMock {}; + let rev_manager = RevisionManager::new(&user_id, &object_id, persistence, compress, snapshot); + Self { + rev_manager: Arc::new(rev_manager), + } + } + pub async fn run_scripts(&self, scripts: Vec) { + for script in scripts { + self.run_script(script).await; + } + } + pub async fn run_script(&self, script: RevisionScript) { + match script { + RevisionScript::AddLocalRevision(revision) => { + self.rev_manager.add_local_revision(&revision).await.unwrap(); + } + RevisionScript::AckRevision { rev_id } => { + // + self.rev_manager.ack_revision(rev_id).await.unwrap() + } + RevisionScript::AssertNextSyncRevisionId { rev_id } => { + // + assert_eq!(self.rev_manager.rev_id(), rev_id) + } + RevisionScript::AssertNextSyncRevision(expected) => { + let next_revision = self.rev_manager.next_sync_revision().await.unwrap(); + assert_eq!(next_revision, expected); + } + } + } +} + +pub struct RevisionDiskCacheMock {} + +impl RevisionDiskCacheMock { + pub fn new() -> Self { + Self {} + } +} + +impl RevisionDiskCache for RevisionDiskCacheMock { + type Error = FlowyError; + + fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { + todo!() + } + + fn get_connection(&self) -> Result { + todo!() + } + + fn read_revision_records( + &self, + object_id: &str, + rev_ids: Option>, + ) -> Result, Self::Error> { + todo!() + } + + fn read_revision_records_with_range( + &self, + object_id: &str, + range: &RevisionRange, + ) -> Result, Self::Error> { + todo!() + } + + fn update_revision_record(&self, changesets: Vec) -> FlowyResult<()> { + todo!() + } + + fn delete_revision_records(&self, object_id: &str, rev_ids: Option>) -> Result<(), Self::Error> { + todo!() + } + + fn delete_and_insert_records( + &self, + object_id: &str, + deleted_rev_ids: Option>, + inserted_records: Vec, + ) -> Result<(), Self::Error> { + todo!() + } +} + +pub struct RevisionConnectionMock {} + +pub struct RevisionSnapshotMock {} + +impl RevisionSnapshotDiskCache for RevisionSnapshotMock { + fn write_snapshot(&self, object_id: &str, rev_id: i64, data: Vec) -> FlowyResult<()> { + todo!() + } + + fn read_snapshot(&self, object_id: &str, rev_id: i64) -> FlowyResult { + todo!() + } +} + +pub struct RevisionCompressMock {} + +impl RevisionCompress for RevisionCompressMock { + fn combine_revisions(&self, revisions: Vec) -> FlowyResult { + todo!() + } +} + +pub struct RevisionMock {} + +// impl std::convert::From for Revision { +// fn from(_: RevisionMock) -> Self {} +// } diff --git a/shared-lib/flowy-sync/src/entities/revision.rs b/shared-lib/flowy-sync/src/entities/revision.rs index 950e8956db..e8384b1243 100644 --- a/shared-lib/flowy-sync/src/entities/revision.rs +++ b/shared-lib/flowy-sync/src/entities/revision.rs @@ -21,12 +21,11 @@ pub struct Revision { #[pb(index = 5)] pub object_id: String, - - #[pb(index = 6)] - ty: RevType, // Deprecated - - #[pb(index = 7)] - pub user_id: String, + // #[pb(index = 6)] + // ty: RevType, // Deprecated + // + // #[pb(index = 7)] + // pub user_id: String, } impl std::convert::From> for Revision { @@ -42,10 +41,9 @@ impl Revision { base_rev_id: i64, rev_id: i64, bytes: Bytes, - user_id: &str, + _user_id: &str, md5: T, ) -> Revision { - let user_id = user_id.to_owned(); let object_id = object_id.to_owned(); let bytes = bytes.to_vec(); let base_rev_id = base_rev_id; @@ -61,8 +59,6 @@ impl Revision { bytes, md5: md5.into(), object_id, - ty: RevType::DeprecatedLocal, - user_id, } } pub fn is_empty(&self) -> bool { diff --git a/shared-lib/lib-ot/tests/node/serde_test.rs b/shared-lib/lib-ot/tests/node/serde_test.rs index 6851cd4164..b3a76d06d2 100644 --- a/shared-lib/lib-ot/tests/node/serde_test.rs +++ b/shared-lib/lib-ot/tests/node/serde_test.rs @@ -1,6 +1,4 @@ -use lib_ot::core::{ - AttributeBuilder, Changeset, NodeData, NodeDataBuilder, NodeOperation, NodeTree, NodeTreeContext, Path, -}; +use lib_ot::core::{AttributeBuilder, Changeset, NodeData, NodeDataBuilder, NodeOperation, NodeTree, Path}; use lib_ot::text_delta::DeltaTextOperationBuilder; #[test] From 2c71e4f885acb02f729fc70498cfa86552cb7851 Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 2 Nov 2022 17:15:27 +0800 Subject: [PATCH 3/9] chore: add tests --- frontend/rust-lib/Cargo.lock | 1 + .../flowy-document/src/editor/queue.rs | 3 +- .../rust-lib/flowy-document/src/manager.rs | 10 +- .../flowy-document/src/old_editor/queue.rs | 3 +- .../flowy-document/src/services/migration.rs | 2 +- .../rev_sqlite/document_rev_sqlite_v0.rs | 21 +-- .../rev_sqlite/document_rev_sqlite_v1.rs | 1 - .../src/services/folder_editor.rs | 9 +- .../src/services/persistence/mod.rs | 2 +- .../rev_sqlite/folder_rev_sqlite.rs | 21 +-- frontend/rust-lib/flowy-grid/src/manager.rs | 6 +- .../flowy-grid/src/services/block_editor.rs | 10 +- .../flowy-grid/src/services/grid_editor.rs | 10 +- .../src/services/grid_view_editor.rs | 2 +- .../persistence/rev_sqlite/grid_block_impl.rs | 1 - .../persistence/rev_sqlite/grid_impl.rs | 1 - .../persistence/rev_sqlite/grid_view_impl.rs | 1 - frontend/rust-lib/flowy-revision/Cargo.toml | 3 + .../flowy-revision/src/cache/reset.rs | 2 +- .../flowy-revision/src/conflict_resolve.rs | 4 +- .../flowy-revision/src/rev_manager.rs | 19 ++- .../flowy-revision/src/rev_persistence.rs | 4 + .../revision_test/local_revision_test.rs | 123 ++++++++++++++++ .../flowy-revision/tests/revision_test/mod.rs | 2 +- .../revision_test/revision_order_test.rs | 8 -- .../tests/revision_test/script.rs | 132 ++++++++++++++++-- .../flowy-sdk/src/deps_resolve/folder_deps.rs | 6 +- .../src/client_grid/block_revision_pad.rs | 2 +- .../src/client_grid/grid_revision_pad.rs | 2 +- .../flowy-sync/src/entities/revision.rs | 33 +---- 30 files changed, 294 insertions(+), 150 deletions(-) create mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs delete mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index af9a252f57..a8871e6302 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -1077,6 +1077,7 @@ dependencies = [ "lib-infra", "lib-ws", "nanoid", + "parking_lot 0.11.2", "serde", "serde_json", "strum", diff --git a/frontend/rust-lib/flowy-document/src/editor/queue.rs b/frontend/rust-lib/flowy-document/src/editor/queue.rs index 24448c9bca..658aebaa72 100644 --- a/frontend/rust-lib/flowy-document/src/editor/queue.rs +++ b/frontend/rust-lib/flowy-document/src/editor/queue.rs @@ -79,8 +79,7 @@ impl DocumentQueue { async fn save_local_operations(&self, transaction: Transaction, md5: String) -> Result { let bytes = Bytes::from(transaction.to_bytes()?); let (base_rev_id, rev_id) = self.rev_manager.next_rev_id_pair(); - let user_id = self.user.user_id()?; - let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, bytes, &user_id, md5); + let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, bytes, md5); let _ = self.rev_manager.add_local_revision(&revision).await?; Ok(rev_id.into()) } diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index 658e3e0a99..104eed23dc 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -291,7 +291,6 @@ impl RevisionCloudService for DocumentRevisionCloudService { let params: DocumentIdPB = object_id.to_string().into(); let server = self.server.clone(); let token = self.token.clone(); - let user_id = user_id.to_string(); FutureResult::new(async move { match server.fetch_document(&token, params).await? { @@ -299,14 +298,7 @@ impl RevisionCloudService for DocumentRevisionCloudService { Some(payload) => { let bytes = Bytes::from(payload.content.clone()); let doc_md5 = md5(&bytes); - let revision = Revision::new( - &payload.doc_id, - payload.base_rev_id, - payload.rev_id, - bytes, - &user_id, - doc_md5, - ); + let revision = Revision::new(&payload.doc_id, payload.base_rev_id, payload.rev_id, bytes, doc_md5); Ok(vec![revision]) } } diff --git a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs index 063ae4f254..dc99d7ec61 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs @@ -178,8 +178,7 @@ impl EditDocumentQueue { async fn save_local_operations(&self, operations: DeltaTextOperations, md5: String) -> Result { let bytes = operations.json_bytes(); let (base_rev_id, rev_id) = self.rev_manager.next_rev_id_pair(); - let user_id = self.user.user_id()?; - let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, bytes, &user_id, md5); + let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, bytes, md5); let _ = self.rev_manager.add_local_revision(&revision).await?; Ok(rev_id.into()) } diff --git a/frontend/rust-lib/flowy-document/src/services/migration.rs b/frontend/rust-lib/flowy-document/src/services/migration.rs index d922486e47..a6b2efcac8 100644 --- a/frontend/rust-lib/flowy-document/src/services/migration.rs +++ b/frontend/rust-lib/flowy-document/src/services/migration.rs @@ -43,7 +43,7 @@ impl DocumentMigration { Ok(transaction) => { let bytes = Bytes::from(transaction.to_bytes()?); let md5 = format!("{:x}", md5::compute(&bytes)); - let revision = Revision::new(&document_id, 0, 1, bytes, &self.user_id, md5); + let revision = Revision::new(&document_id, 0, 1, bytes, md5); let record = SyncRecord::new(revision); match disk_cache.create_revision_records(vec![record]) { Ok(_) => {} diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs index 7a4485861f..9b103d016d 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs @@ -9,7 +9,7 @@ use flowy_database::{ use flowy_error::{internal_error, FlowyError, FlowyResult}; use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ - entities::revision::{RevType, Revision, RevisionRange}, + entities::revision::{Revision, RevisionRange}, util::md5, }; use std::collections::HashMap; @@ -251,7 +251,6 @@ fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRec table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { @@ -288,21 +287,3 @@ impl std::convert::From for RevTableType { } } } - -impl std::convert::From for RevTableType { - fn from(ty: RevType) -> Self { - match ty { - RevType::DeprecatedLocal => RevTableType::Local, - RevType::DeprecatedRemote => RevTableType::Remote, - } - } -} - -impl std::convert::From for RevType { - fn from(ty: RevTableType) -> Self { - match ty { - RevTableType::Local => RevType::DeprecatedLocal, - RevTableType::Remote => RevType::DeprecatedRemote, - } - } -} diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs index d02528a79f..70c65734de 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs @@ -227,7 +227,6 @@ fn mk_revision_record_from_table(user_id: &str, table: DocumentRevisionTable) -> table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { diff --git a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs index 4a69ca743a..d54df91d0d 100644 --- a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs +++ b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs @@ -83,14 +83,7 @@ impl FolderEditor { let FolderChangeset { operations: delta, md5 } = change; let (base_rev_id, rev_id) = self.rev_manager.next_rev_id_pair(); let delta_data = delta.json_bytes(); - let revision = Revision::new( - &self.rev_manager.object_id, - base_rev_id, - rev_id, - delta_data, - &self.user_id, - md5, - ); + let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, delta_data, md5); let _ = futures::executor::block_on(async { self.rev_manager.add_local_revision(&revision).await })?; Ok(()) } diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs index 147effff9b..edf91695ca 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs @@ -111,7 +111,7 @@ impl FolderPersistence { let pool = self.database.db_pool()?; let json = folder.to_json()?; let delta_data = FolderOperationsBuilder::new().insert(&json).build().json_bytes(); - let revision = Revision::initial_revision(user_id, folder_id.as_ref(), delta_data); + let revision = Revision::initial_revision(folder_id.as_ref(), delta_data); let record = SyncRecord { revision, state: RevisionState::Sync, diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs index c094d4be8c..13f9a4e48c 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs @@ -9,7 +9,7 @@ use flowy_database::{ use flowy_error::{internal_error, FlowyError, FlowyResult}; use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionState, SyncRecord}; use flowy_sync::{ - entities::revision::{RevType, Revision, RevisionRange}, + entities::revision::{Revision, RevisionRange}, util::md5, }; @@ -227,7 +227,6 @@ fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRec table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { @@ -264,21 +263,3 @@ impl std::convert::From for RevTableType { } } } - -impl std::convert::From for RevTableType { - fn from(ty: RevType) -> Self { - match ty { - RevType::DeprecatedLocal => RevTableType::Local, - RevType::DeprecatedRemote => RevTableType::Remote, - } - } -} - -impl std::convert::From for RevType { - fn from(ty: RevTableType) -> Self { - match ty { - RevTableType::Local => RevType::DeprecatedLocal, - RevTableType::Remote => RevType::DeprecatedRemote, - } - } -} diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index 72c6e808dc..d3e28e3725 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -208,7 +208,7 @@ pub async fn make_grid_view_data( // Create grid's block let grid_block_delta = make_grid_block_operations(block_meta_data); let block_delta_data = grid_block_delta.json_bytes(); - let revision = Revision::initial_revision(user_id, block_id, block_delta_data); + let revision = Revision::initial_revision(block_id, block_delta_data); let _ = grid_manager.create_grid_block(&block_id, vec![revision]).await?; } @@ -219,7 +219,7 @@ pub async fn make_grid_view_data( // Create grid let grid_rev_delta = make_grid_operations(&grid_rev); let grid_rev_delta_bytes = grid_rev_delta.json_bytes(); - let revision = Revision::initial_revision(user_id, &grid_id, grid_rev_delta_bytes.clone()); + let revision = Revision::initial_revision(&grid_id, grid_rev_delta_bytes.clone()); let _ = grid_manager.create_grid(&grid_id, vec![revision]).await?; // Create grid view @@ -230,7 +230,7 @@ pub async fn make_grid_view_data( }; let grid_view_delta = make_grid_view_operations(&grid_view); let grid_view_delta_bytes = grid_view_delta.json_bytes(); - let revision = Revision::initial_revision(user_id, view_id, grid_view_delta_bytes); + let revision = Revision::initial_revision(view_id, grid_view_delta_bytes); let _ = grid_manager.create_grid_view(view_id, vec![revision]).await?; Ok(grid_rev_delta_bytes) diff --git a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs index d5e01a15a3..329f8b0824 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs @@ -167,17 +167,9 @@ impl GridBlockRevisionEditor { async fn apply_change(&self, change: GridBlockRevisionChangeset) -> FlowyResult<()> { let GridBlockRevisionChangeset { operations: delta, md5 } = change; - let user_id = self.user_id.clone(); let (base_rev_id, rev_id) = self.rev_manager.next_rev_id_pair(); let delta_data = delta.json_bytes(); - let revision = Revision::new( - &self.rev_manager.object_id, - base_rev_id, - rev_id, - delta_data, - &user_id, - md5, - ); + let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, delta_data, md5); let _ = self.rev_manager.add_local_revision(&revision).await?; Ok(()) } diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs index 3e6a9d329e..5d84414b7a 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs @@ -757,17 +757,9 @@ impl GridRevisionEditor { async fn apply_change(&self, change: GridRevisionChangeset) -> FlowyResult<()> { let GridRevisionChangeset { operations: delta, md5 } = change; - let user_id = self.user.user_id()?; let (base_rev_id, rev_id) = self.rev_manager.next_rev_id_pair(); let delta_data = delta.json_bytes(); - let revision = Revision::new( - &self.rev_manager.object_id, - base_rev_id, - rev_id, - delta_data, - &user_id, - md5, - ); + let revision = Revision::new(&self.rev_manager.object_id, base_rev_id, rev_id, delta_data, md5); let _ = self.rev_manager.add_local_revision(&revision).await?; Ok(()) } diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs index 5a62698e87..767d6ae6dc 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs @@ -461,7 +461,7 @@ async fn apply_change( let GridViewRevisionChangeset { operations: delta, md5 } = change; let (base_rev_id, rev_id) = rev_manager.next_rev_id_pair(); let delta_data = delta.json_bytes(); - let revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, delta_data, user_id, md5); + let revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, delta_data, md5); let _ = rev_manager.add_local_revision(&revision).await?; Ok(()) } diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs index a4895ae5e8..31cb05bb3a 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs @@ -226,7 +226,6 @@ fn mk_revision_record_from_table(user_id: &str, table: GridBlockRevisionTable) - table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs index 3fd121c5e3..e1ab04854c 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs @@ -224,7 +224,6 @@ fn mk_revision_record_from_table(user_id: &str, table: GridRevisionTable) -> Syn table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs index fe9490d0de..d86451412c 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs @@ -226,7 +226,6 @@ fn mk_revision_record_from_table(user_id: &str, table: GridViewRevisionTable) -> table.base_rev_id, table.rev_id, Bytes::from(table.data), - user_id, md5, ); SyncRecord { diff --git a/frontend/rust-lib/flowy-revision/Cargo.toml b/frontend/rust-lib/flowy-revision/Cargo.toml index 2599e13a75..54f1902afa 100644 --- a/frontend/rust-lib/flowy-revision/Cargo.toml +++ b/frontend/rust-lib/flowy-revision/Cargo.toml @@ -23,6 +23,9 @@ serde_json = {version = "1.0"} [dev-dependencies] nanoid = "0.4.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +parking_lot = "0.11" [features] flowy_unit_test = [] \ No newline at end of file diff --git a/frontend/rust-lib/flowy-revision/src/cache/reset.rs b/frontend/rust-lib/flowy-revision/src/cache/reset.rs index 6e6872329b..c4dc8ae970 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/reset.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/reset.rs @@ -75,7 +75,7 @@ where .await?; let bytes = self.target.reset_data(revisions)?; - let revision = Revision::initial_revision(&self.user_id, self.target.target_id(), bytes); + let revision = Revision::initial_revision(self.target.target_id(), bytes); let record = SyncRecord::new(revision); tracing::trace!("Reset {} revision record object", self.target.target_id()); diff --git a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs index 4fb8260311..2700cc4f1f 100644 --- a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs +++ b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs @@ -163,13 +163,13 @@ where { let (base_rev_id, rev_id) = rev_manager.next_rev_id_pair(); let bytes = client_operations.serialize_operations(); - let client_revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, bytes, user_id, md5.clone()); + let client_revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, bytes, md5.clone()); match server_operations { None => (client_revision, None), Some(operations) => { let bytes = operations.serialize_operations(); - let server_revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, bytes, user_id, md5); + let server_revision = Revision::new(&rev_manager.object_id, base_rev_id, rev_id, bytes, md5); (client_revision, Some(server_revision)) } } diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index e8bab34233..32affc3200 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -63,12 +63,24 @@ pub trait RevisionCompress: Send + Sync { let (base_rev_id, rev_id) = first_revision.pair_rev_id(); let md5 = last_revision.md5.clone(); let bytes = self.combine_revisions(revisions)?; - Ok(Revision::new(object_id, base_rev_id, rev_id, bytes, user_id, md5)) + Ok(Revision::new(object_id, base_rev_id, rev_id, bytes, md5)) } fn combine_revisions(&self, revisions: Vec) -> FlowyResult; } +pub struct RevisionConfiguration { + merge_when_excess_number_of_version: i64, +} + +impl std::default::Default for RevisionConfiguration { + fn default() -> Self { + Self { + merge_when_excess_number_of_version: 100, + } + } +} + pub struct RevisionManager { pub object_id: String, user_id: String, @@ -79,6 +91,7 @@ pub struct RevisionManager { rev_compress: Arc, #[cfg(feature = "flowy_unit_test")] rev_ack_notifier: tokio::sync::broadcast::Sender, + // configuration: RevisionConfiguration, } impl RevisionManager { @@ -190,6 +203,10 @@ impl RevisionManager { self.rev_id_counter.value() } + pub async fn next_sync_rev_id(&self) -> Option { + self.rev_persistence.next_sync_rev_id().await + } + pub fn next_rev_id_pair(&self) -> (i64, i64) { let cur = self.rev_id_counter.value(); let next = self.rev_id_counter.next_id(); diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index e0c9529c1f..57988ca90e 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -128,6 +128,10 @@ where } } + pub(crate) async fn next_sync_rev_id(&self) -> Option { + self.sync_seq.read().await.next_rev_id() + } + /// The cache gets reset while it conflicts with the remote revisions. #[tracing::instrument(level = "trace", skip(self, revisions), err)] pub(crate) async fn reset(&self, revisions: Vec) -> FlowyResult<()> { diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs new file mode 100644 index 0000000000..4c3aa97ae0 --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs @@ -0,0 +1,123 @@ +use crate::revision_test::script::{RevisionScript::*, RevisionTest}; +use flowy_revision::REVISION_WRITE_INTERVAL_IN_MILLIS; + +#[tokio::test] +async fn revision_sync_test() { + let test = RevisionTest::new().await; + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + + test.run_script(AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id, + }) + .await; + + test.run_script(AssertNextSyncRevisionId { rev_id: Some(rev_id) }).await; + test.run_script(AckRevision { rev_id }).await; + test.run_script(AssertNextSyncRevisionId { rev_id: None }).await; +} + +#[tokio::test] +async fn revision_sync_multiple_revisions() { + let test = RevisionTest::new().await; + let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); + + test.run_script(AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id: rev_id_1, + }) + .await; + + let (base_rev_id, rev_id_2) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "456".to_string(), + base_rev_id, + rev_id: rev_id_2, + }) + .await; + + test.run_scripts(vec![ + AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, + AckRevision { rev_id: rev_id_1 }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_2) }, + AckRevision { rev_id: rev_id_2 }, + AssertNextSyncRevisionId { rev_id: None }, + ]) + .await; +} + +#[tokio::test] +async fn revision_compress_two_revisions_test() { + let test = RevisionTest::new().await; + let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); + + test.run_script(AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id: rev_id_1, + }) + .await; + + // rev_id_2 will be merged with rev_id_3 + let (base_rev_id, rev_id_2) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "456".to_string(), + base_rev_id, + rev_id: rev_id_2, + }) + .await; + + let (base_rev_id, rev_id_3) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "789".to_string(), + base_rev_id, + rev_id: rev_id_3, + }) + .await; + + test.run_scripts(vec![ + Wait { + milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, + }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, + AckRevision { rev_id: rev_id_1 }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_2) }, + AssertNextSyncRevisionContent { + expected: "456789".to_string(), + }, + ]) + .await; +} + +#[tokio::test] +async fn revision_compress_multiple_revisions_test() { + let test = RevisionTest::new().await; + let mut expected = "".to_owned(); + + for i in 0..100 { + let content = format!("{}", i); + if i != 0 { + expected.push_str(&content); + } + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content, + base_rev_id, + rev_id, + }) + .await; + } + + test.run_scripts(vec![ + Wait { + milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, + }, + AssertNextSyncRevisionId { rev_id: Some(1) }, + AckRevision { rev_id: 1 }, + AssertNextSyncRevisionId { rev_id: Some(2) }, + AssertNextSyncRevisionContent { expected }, + ]) + .await; +} diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs index 779735a23d..91300b4b71 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs @@ -1,2 +1,2 @@ -mod revision_order_test; +mod local_revision_test; mod script; diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs deleted file mode 100644 index a7970b875c..0000000000 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_order_test.rs +++ /dev/null @@ -1,8 +0,0 @@ -use crate::revision_test::script::{RevisionScript::*, RevisionTest}; - -#[tokio::test] -async fn test() { - let test = RevisionTest::new().await; - let scripts = vec![]; - test.run_scripts(scripts).await; -} diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs index f2dcc4846a..8aeba313cf 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -5,13 +5,33 @@ use flowy_revision::{ RevisionCompress, RevisionManager, RevisionPersistence, RevisionSnapshotDiskCache, RevisionSnapshotInfo, }; use flowy_sync::entities::revision::{Revision, RevisionRange}; +use flowy_sync::util::md5; use nanoid::nanoid; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; use std::sync::Arc; +use std::time::Duration; +use tokio::time::interval; pub enum RevisionScript { - AddLocalRevision(Revision), - AckRevision { rev_id: i64 }, - AssertNextSyncRevisionId { rev_id: i64 }, + AddLocalRevision { + content: String, + base_rev_id: i64, + rev_id: i64, + }, + AckRevision { + rev_id: i64, + }, + AssertNextSyncRevisionId { + rev_id: Option, + }, + AssertNextSyncRevisionContent { + expected: String, + }, + Wait { + milliseconds: u64, + }, + AssertNextSyncRevision(Option), } @@ -36,9 +56,28 @@ impl RevisionTest { self.run_script(script).await; } } + + pub fn next_rev_id_pair(&self) -> (i64, i64) { + self.rev_manager.next_rev_id_pair() + } + pub async fn run_script(&self, script: RevisionScript) { match script { - RevisionScript::AddLocalRevision(revision) => { + RevisionScript::AddLocalRevision { + content, + base_rev_id, + rev_id, + } => { + let object = RevisionObjectMock::new(&content); + let bytes = object.to_bytes(); + let md5 = md5(&bytes); + let revision = Revision::new( + &self.rev_manager.object_id, + base_rev_id, + rev_id, + Bytes::from(bytes), + md5, + ); self.rev_manager.add_local_revision(&revision).await.unwrap(); } RevisionScript::AckRevision { rev_id } => { @@ -46,8 +85,19 @@ impl RevisionTest { self.rev_manager.ack_revision(rev_id).await.unwrap() } RevisionScript::AssertNextSyncRevisionId { rev_id } => { + assert_eq!(self.rev_manager.next_sync_rev_id().await, rev_id) + } + RevisionScript::AssertNextSyncRevisionContent { expected } => { // - assert_eq!(self.rev_manager.rev_id(), rev_id) + let rev_id = self.rev_manager.next_sync_rev_id().await.unwrap(); + let revision = self.rev_manager.get_revision(rev_id).await.unwrap(); + let object = RevisionObjectMock::from_bytes(&revision.bytes); + assert_eq!(object.content, expected); + } + RevisionScript::Wait { milliseconds } => { + // let mut interval = interval(Duration::from_millis(milliseconds)); + // interval.tick().await; + tokio::time::sleep(Duration::from_millis(milliseconds)).await; } RevisionScript::AssertNextSyncRevision(expected) => { let next_revision = self.rev_manager.next_sync_revision().await.unwrap(); @@ -57,11 +107,15 @@ impl RevisionTest { } } -pub struct RevisionDiskCacheMock {} +pub struct RevisionDiskCacheMock { + records: RwLock>, +} impl RevisionDiskCacheMock { pub fn new() -> Self { - Self {} + Self { + records: RwLock::new(vec![]), + } } } @@ -69,7 +123,8 @@ impl RevisionDiskCache for RevisionDiskCacheMock { type Error = FlowyError; fn create_revision_records(&self, revision_records: Vec) -> Result<(), Self::Error> { - todo!() + self.records.write().extend(revision_records); + Ok(()) } fn get_connection(&self) -> Result { @@ -93,11 +148,36 @@ impl RevisionDiskCache for RevisionDiskCacheMock { } fn update_revision_record(&self, changesets: Vec) -> FlowyResult<()> { - todo!() + for changeset in changesets { + if let Some(record) = self + .records + .write() + .iter_mut() + .find(|record| record.revision.rev_id == *changeset.rev_id.as_ref()) + { + record.state = changeset.state; + } + } + Ok(()) } fn delete_revision_records(&self, object_id: &str, rev_ids: Option>) -> Result<(), Self::Error> { - todo!() + match rev_ids { + None => {} + Some(rev_ids) => { + for rev_id in rev_ids { + if let Some(index) = self + .records + .read() + .iter() + .position(|record| record.revision.rev_id == rev_id) + { + self.records.write().remove(index); + } + } + } + } + Ok(()) } fn delete_and_insert_records( @@ -128,12 +208,34 @@ pub struct RevisionCompressMock {} impl RevisionCompress for RevisionCompressMock { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { - todo!() + let mut object = RevisionObjectMock::new(""); + for revision in revisions { + let other = RevisionObjectMock::from_bytes(&revision.bytes); + object.compose(other); + } + Ok(Bytes::from(object.to_bytes())) } } -pub struct RevisionMock {} +#[derive(Serialize, Deserialize)] +pub struct RevisionObjectMock { + content: String, +} -// impl std::convert::From for Revision { -// fn from(_: RevisionMock) -> Self {} -// } +impl RevisionObjectMock { + pub fn new(s: &str) -> Self { + Self { content: s.to_owned() } + } + + pub fn compose(&mut self, other: RevisionObjectMock) { + self.content.push_str(other.content.as_str()); + } + + pub fn to_bytes(&self) -> Vec { + serde_json::to_vec(self).unwrap() + } + + pub fn from_bytes(bytes: &[u8]) -> Self { + serde_json::from_slice(bytes).unwrap() + } +} diff --git a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs index 0320d078e9..33e588f99f 100644 --- a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs +++ b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs @@ -151,7 +151,7 @@ impl ViewDataProcessor for DocumentViewDataProcessor { ) -> FutureResult<(), FlowyError> { // Only accept Document type debug_assert_eq!(layout, ViewLayoutTypePB::Document); - let revision = Revision::initial_revision(user_id, view_id, view_data); + let revision = Revision::initial_revision(view_id, view_data); let view_id = view_id.to_string(); let manager = self.0.clone(); @@ -194,7 +194,7 @@ impl ViewDataProcessor for DocumentViewDataProcessor { let document_content = self.0.initial_document_content(); FutureResult::new(async move { let delta_data = Bytes::from(document_content); - let revision = Revision::initial_revision(&user_id, &view_id, delta_data.clone()); + let revision = Revision::initial_revision(&view_id, delta_data.clone()); let _ = manager.create_document(view_id, vec![revision]).await?; Ok(delta_data) }) @@ -225,7 +225,7 @@ impl ViewDataProcessor for GridViewDataProcessor { _layout: ViewLayoutTypePB, delta_data: Bytes, ) -> FutureResult<(), FlowyError> { - let revision = Revision::initial_revision(user_id, view_id, delta_data); + let revision = Revision::initial_revision(view_id, delta_data); let view_id = view_id.to_string(); let grid_manager = self.0.clone(); FutureResult::new(async move { diff --git a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs index c5a504c358..ebf578cc3c 100644 --- a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs @@ -259,7 +259,7 @@ pub fn make_grid_block_operations(block_rev: &GridBlockRevision) -> GridBlockOpe pub fn make_grid_block_revisions(user_id: &str, grid_block_meta_data: &GridBlockRevision) -> RepeatedRevision { let operations = make_grid_block_operations(grid_block_meta_data); let bytes = operations.json_bytes(); - let revision = Revision::initial_revision(user_id, &grid_block_meta_data.block_id, bytes); + let revision = Revision::initial_revision(&grid_block_meta_data.block_id, bytes); revision.into() } diff --git a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs index 6f99a23e53..5f3d7860ae 100644 --- a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs @@ -412,7 +412,7 @@ pub fn make_grid_operations(grid_rev: &GridRevision) -> GridOperations { pub fn make_grid_revisions(user_id: &str, grid_rev: &GridRevision) -> RepeatedRevision { let operations = make_grid_operations(grid_rev); let bytes = operations.json_bytes(); - let revision = Revision::initial_revision(user_id, &grid_rev.grid_id, bytes); + let revision = Revision::initial_revision(&grid_rev.grid_id, bytes); revision.into() } diff --git a/shared-lib/flowy-sync/src/entities/revision.rs b/shared-lib/flowy-sync/src/entities/revision.rs index e8384b1243..e8b7e42d88 100644 --- a/shared-lib/flowy-sync/src/entities/revision.rs +++ b/shared-lib/flowy-sync/src/entities/revision.rs @@ -1,6 +1,6 @@ use crate::util::md5; use bytes::Bytes; -use flowy_derive::{ProtoBuf, ProtoBuf_Enum}; +use flowy_derive::ProtoBuf; use std::{convert::TryFrom, fmt::Formatter, ops::RangeInclusive}; pub type RevisionObject = lib_ot::text_delta::DeltaTextOperations; @@ -21,11 +21,6 @@ pub struct Revision { #[pb(index = 5)] pub object_id: String, - // #[pb(index = 6)] - // ty: RevType, // Deprecated - // - // #[pb(index = 7)] - // pub user_id: String, } impl std::convert::From> for Revision { @@ -36,14 +31,7 @@ impl std::convert::From> for Revision { } impl Revision { - pub fn new>( - object_id: &str, - base_rev_id: i64, - rev_id: i64, - bytes: Bytes, - _user_id: &str, - md5: T, - ) -> Revision { + pub fn new>(object_id: &str, base_rev_id: i64, rev_id: i64, bytes: Bytes, md5: T) -> Revision { let object_id = object_id.to_owned(); let bytes = bytes.to_vec(); let base_rev_id = base_rev_id; @@ -61,6 +49,7 @@ impl Revision { object_id, } } + pub fn is_empty(&self) -> bool { self.base_rev_id == self.rev_id } @@ -73,9 +62,9 @@ impl Revision { self.rev_id == 0 } - pub fn initial_revision(user_id: &str, object_id: &str, bytes: Bytes) -> Self { + pub fn initial_revision(object_id: &str, bytes: Bytes) -> Self { let md5 = md5(&bytes); - Self::new(object_id, 0, 0, bytes, user_id, md5) + Self::new(object_id, 0, 0, bytes, md5) } } @@ -211,15 +200,3 @@ impl RevisionRange { self.iter().collect::>() } } - -#[derive(Debug, ProtoBuf_Enum, Clone, Eq, PartialEq)] -pub enum RevType { - DeprecatedLocal = 0, - DeprecatedRemote = 1, -} - -impl std::default::Default for RevType { - fn default() -> Self { - RevType::DeprecatedLocal - } -} From ff7aab73cc8c8ce87ed727dd4a7e69e8259ffa1a Mon Sep 17 00:00:00 2001 From: nathan Date: Sun, 6 Nov 2022 09:59:53 +0800 Subject: [PATCH 4/9] chore: merge with config setting --- .../flowy-document/src/editor/document.rs | 4 +- .../rust-lib/flowy-document/src/manager.rs | 9 +- .../flowy-document/src/old_editor/editor.rs | 4 +- .../flowy-document/src/old_editor/queue.rs | 1 + .../rev_sqlite/document_rev_sqlite_v0.rs | 2 +- .../rev_sqlite/document_rev_sqlite_v1.rs | 2 +- frontend/rust-lib/flowy-folder/src/manager.rs | 8 +- .../src/services/folder_editor.rs | 8 +- .../rev_sqlite/folder_rev_sqlite.rs | 2 +- frontend/rust-lib/flowy-grid/src/manager.rs | 13 +- .../flowy-grid/src/services/block_editor.rs | 5 +- .../flowy-grid/src/services/block_manager.rs | 7 +- .../flowy-grid/src/services/grid_editor.rs | 5 +- .../src/services/grid_view_editor.rs | 6 +- .../src/services/grid_view_manager.rs | 7 +- .../persistence/rev_sqlite/grid_block_impl.rs | 2 +- .../persistence/rev_sqlite/grid_impl.rs | 2 +- .../persistence/rev_sqlite/grid_view_impl.rs | 2 +- .../flowy-revision/src/cache/memory.rs | 4 + .../flowy-revision/src/cache/reset.rs | 4 +- .../flowy-revision/src/conflict_resolve.rs | 2 +- .../flowy-revision/src/rev_manager.rs | 30 +--- .../flowy-revision/src/rev_persistence.rs | 165 +++++++++++------- .../revision_test/local_revision_test.rs | 137 ++++++++++++--- .../tests/revision_test/script.rs | 48 ++--- .../flowy-sdk/src/deps_resolve/folder_deps.rs | 6 +- .../src/client_grid/block_revision_pad.rs | 2 +- .../src/client_grid/grid_revision_pad.rs | 2 +- .../flowy-sync/src/entities/revision.rs | 4 +- 29 files changed, 317 insertions(+), 176 deletions(-) diff --git a/frontend/rust-lib/flowy-document/src/editor/document.rs b/frontend/rust-lib/flowy-document/src/editor/document.rs index 8af53a5727..5099313d86 100644 --- a/frontend/rust-lib/flowy-document/src/editor/document.rs +++ b/frontend/rust-lib/flowy-document/src/editor/document.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; -use flowy_revision::{RevisionCompress, RevisionObjectDeserializer, RevisionObjectSerializer}; +use flowy_revision::{RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer}; use flowy_sync::entities::revision::Revision; use lib_ot::core::{Extension, NodeDataBuilder, NodeOperation, NodeTree, NodeTreeContext, Selection, Transaction}; use lib_ot::text_delta::DeltaTextOperationBuilder; @@ -96,7 +96,7 @@ impl RevisionObjectSerializer for DocumentRevisionSerde { } pub(crate) struct DocumentRevisionCompress(); -impl RevisionCompress for DocumentRevisionCompress { +impl RevisionMergeable for DocumentRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { DocumentRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index 104eed23dc..27c5ec5814 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -9,7 +9,8 @@ use dashmap::DashMap; use flowy_database::ConnectionPool; use flowy_error::FlowyResult; use flowy_revision::{ - RevisionCloudService, RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence, + RevisionCloudService, RevisionManager, RevisionPersistence, RevisionPersistenceConfiguration, RevisionWebSocket, + SQLiteRevisionSnapshotPersistence, }; use flowy_sync::client_document::initial_delta_document_content; use flowy_sync::entities::{document::DocumentIdPB, revision::Revision, ws_data::ServerRevisionWSData}; @@ -246,7 +247,8 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDocumentRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); Ok(RevisionManager::new( @@ -266,7 +268,8 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDeltaDocumentRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); Ok(RevisionManager::new( diff --git a/frontend/rust-lib/flowy-document/src/old_editor/editor.rs b/frontend/rust-lib/flowy-document/src/old_editor/editor.rs index f0a40f9dba..35de2de668 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/editor.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/editor.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use flowy_database::ConnectionPool; use flowy_error::{internal_error, FlowyResult}; use flowy_revision::{ - RevisionCloudService, RevisionCompress, RevisionManager, RevisionObjectDeserializer, RevisionObjectSerializer, + RevisionCloudService, RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer, RevisionWebSocket, }; use flowy_sync::entities::ws_data::ServerRevisionWSData; @@ -270,7 +270,7 @@ impl RevisionObjectSerializer for DeltaDocumentRevisionSerde { } pub(crate) struct DeltaDocumentRevisionCompress(); -impl RevisionCompress for DeltaDocumentRevisionCompress { +impl RevisionMergeable for DeltaDocumentRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { DeltaDocumentRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs index dc99d7ec61..85ec445dcc 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/queue.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/queue.rs @@ -23,6 +23,7 @@ use tokio::sync::{oneshot, RwLock}; // serial. pub(crate) struct EditDocumentQueue { document: Arc>, + #[allow(dead_code)] user: Arc, rev_manager: Arc>>, receiver: Option, diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs index 9b103d016d..8ac4b9bae6 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v0.rs @@ -244,7 +244,7 @@ impl std::default::Default for TextRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: RevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.doc_id, diff --git a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs index 70c65734de..041dec8bf3 100644 --- a/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs +++ b/frontend/rust-lib/flowy-document/src/services/persistence/rev_sqlite/document_rev_sqlite_v1.rs @@ -220,7 +220,7 @@ impl std::default::Default for DocumentRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: DocumentRevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: DocumentRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.document_id, diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs index 3db8c97a52..c942c79a9f 100644 --- a/frontend/rust-lib/flowy-folder/src/manager.rs +++ b/frontend/rust-lib/flowy-folder/src/manager.rs @@ -15,7 +15,10 @@ use bytes::Bytes; use flowy_document::editor::initial_read_me; use flowy_error::FlowyError; use flowy_folder_data_model::user_default; -use flowy_revision::{RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence}; +use flowy_revision::{ + RevisionManager, RevisionPersistence, RevisionPersistenceConfiguration, RevisionWebSocket, + SQLiteRevisionSnapshotPersistence, +}; use flowy_sync::{client_folder::FolderPad, entities::ws_data::ServerRevisionWSData}; use lazy_static::lazy_static; use lib_infra::future::FutureResult; @@ -165,7 +168,8 @@ impl FolderManager { let pool = self.persistence.db_pool()?; let object_id = folder_id.as_ref(); let disk_cache = SQLiteFolderRevisionPersistence::new(user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::new(50); + let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache, configuration); let rev_compactor = FolderRevisionCompress(); // let history_persistence = SQLiteRevisionHistoryPersistence::new(object_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(object_id, pool); diff --git a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs index d54df91d0d..93ec061565 100644 --- a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs +++ b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs @@ -2,7 +2,7 @@ use crate::manager::FolderId; use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_revision::{ - RevisionCloudService, RevisionCompress, RevisionManager, RevisionObjectDeserializer, RevisionObjectSerializer, + RevisionCloudService, RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer, RevisionWebSocket, }; use flowy_sync::util::make_operations_from_revisions; @@ -18,9 +18,8 @@ use parking_lot::RwLock; use std::sync::Arc; pub struct FolderEditor { - user_id: String, #[allow(dead_code)] - pub(crate) folder_id: FolderId, + user_id: String, pub(crate) folder: Arc>, rev_manager: Arc>>, #[cfg(feature = "sync")] @@ -56,7 +55,6 @@ impl FolderEditor { let folder_id = folder_id.to_owned(); Ok(Self { user_id, - folder_id, folder, rev_manager, #[cfg(feature = "sync")] @@ -113,7 +111,7 @@ impl RevisionObjectSerializer for FolderRevisionSerde { } pub struct FolderRevisionCompress(); -impl RevisionCompress for FolderRevisionCompress { +impl RevisionMergeable for FolderRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { FolderRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs index 13f9a4e48c..bf6385f978 100644 --- a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs +++ b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs @@ -220,7 +220,7 @@ impl std::default::Default for TextRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: RevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.doc_id, diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index d3e28e3725..f1f4824775 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -13,7 +13,10 @@ use dashmap::DashMap; use flowy_database::ConnectionPool; use flowy_error::{FlowyError, FlowyResult}; use flowy_grid_data_model::revision::{BuildGridContext, GridRevision, GridViewRevision}; -use flowy_revision::{RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence}; +use flowy_revision::{ + RevisionManager, RevisionPersistence, RevisionPersistenceConfiguration, RevisionWebSocket, + SQLiteRevisionSnapshotPersistence, +}; use flowy_sync::client_grid::{make_grid_block_operations, make_grid_operations, make_grid_view_operations}; use flowy_sync::entities::revision::Revision; use std::sync::Arc; @@ -161,7 +164,8 @@ impl GridManager { ) -> FlowyResult>> { let user_id = self.grid_user.user_id()?; let disk_cache = SQLiteGridRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, grid_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, grid_id, disk_cache, configuration); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(grid_id, pool); let rev_compactor = GridRevisionCompress(); let rev_manager = RevisionManager::new(&user_id, grid_id, rev_persistence, rev_compactor, snapshot_persistence); @@ -175,7 +179,8 @@ impl GridManager { ) -> FlowyResult>> { let user_id = self.grid_user.user_id()?; let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); let rev_compactor = GridBlockRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); let rev_manager = @@ -185,7 +190,7 @@ impl GridManager { } pub async fn make_grid_view_data( - user_id: &str, + _user_id: &str, view_id: &str, layout: GridLayout, grid_manager: Arc, diff --git a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs index 329f8b0824..2a8d01cb54 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_grid_data_model::revision::{CellRevision, GridBlockRevision, RowChangeset, RowRevision}; use flowy_revision::{ - RevisionCloudService, RevisionCompress, RevisionManager, RevisionObjectDeserializer, RevisionObjectSerializer, + RevisionCloudService, RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer, }; use flowy_sync::client_grid::{GridBlockRevisionChangeset, GridBlockRevisionPad}; use flowy_sync::entities::revision::Revision; @@ -17,6 +17,7 @@ use std::sync::Arc; use tokio::sync::RwLock; pub struct GridBlockRevisionEditor { + #[allow(dead_code)] user_id: String, pub block_id: String, pad: Arc>, @@ -204,7 +205,7 @@ impl RevisionObjectSerializer for GridBlockRevisionSerde { } pub struct GridBlockRevisionCompress(); -impl RevisionCompress for GridBlockRevisionCompress { +impl RevisionMergeable for GridBlockRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { GridBlockRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs index f9b85b7e03..a95fdb5e41 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs @@ -10,7 +10,9 @@ use flowy_error::FlowyResult; use flowy_grid_data_model::revision::{ GridBlockMetaRevision, GridBlockMetaRevisionChangeset, RowChangeset, RowRevision, }; -use flowy_revision::{RevisionManager, RevisionPersistence, SQLiteRevisionSnapshotPersistence}; +use flowy_revision::{ + RevisionManager, RevisionPersistence, RevisionPersistenceConfiguration, SQLiteRevisionSnapshotPersistence, +}; use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; @@ -273,7 +275,8 @@ async fn make_block_editor(user: &Arc, block_id: &str) -> FlowyRes let pool = user.db_pool()?; let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); let rev_compactor = GridBlockRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); let rev_manager = RevisionManager::new(&user_id, block_id, rev_persistence, rev_compactor, snapshot_persistence); diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs index 5d84414b7a..20e8cf34cb 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs @@ -17,7 +17,7 @@ use bytes::Bytes; use flowy_error::{ErrorCode, FlowyError, FlowyResult}; use flowy_grid_data_model::revision::*; use flowy_revision::{ - RevisionCloudService, RevisionCompress, RevisionManager, RevisionObjectDeserializer, RevisionObjectSerializer, + RevisionCloudService, RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer, }; use flowy_sync::client_grid::{GridRevisionChangeset, GridRevisionPad, JsonDeserializer}; use flowy_sync::entities::revision::Revision; @@ -33,6 +33,7 @@ use tokio::sync::RwLock; pub struct GridRevisionEditor { pub grid_id: String, + #[allow(dead_code)] user: Arc, grid_pad: Arc>, view_manager: Arc, @@ -846,7 +847,7 @@ impl RevisionCloudService for GridRevisionCloudService { pub struct GridRevisionCompress(); -impl RevisionCompress for GridRevisionCompress { +impl RevisionMergeable for GridRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { GridRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs index 767d6ae6dc..4be9043861 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs @@ -19,7 +19,7 @@ use flowy_grid_data_model::revision::{ RowChangeset, RowRevision, }; use flowy_revision::{ - RevisionCloudService, RevisionCompress, RevisionManager, RevisionObjectDeserializer, RevisionObjectSerializer, + RevisionCloudService, RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionObjectSerializer, }; use flowy_sync::client_grid::{GridViewRevisionChangeset, GridViewRevisionPad}; use flowy_sync::entities::revision::Revision; @@ -454,7 +454,7 @@ async fn new_group_controller_with_field_rev( } async fn apply_change( - user_id: &str, + _user_id: &str, rev_manager: Arc>>, change: GridViewRevisionChangeset, ) -> FlowyResult<()> { @@ -496,7 +496,7 @@ impl RevisionObjectSerializer for GridViewRevisionSerde { } pub struct GridViewRevisionCompress(); -impl RevisionCompress for GridViewRevisionCompress { +impl RevisionMergeable for GridViewRevisionCompress { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { GridViewRevisionSerde::combine_revisions(revisions) } diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs index 2f3b90d0ab..6865b3bd32 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs @@ -11,7 +11,9 @@ use dashmap::DashMap; use flowy_database::ConnectionPool; use flowy_error::FlowyResult; use flowy_grid_data_model::revision::{FieldRevision, RowChangeset, RowRevision}; -use flowy_revision::{RevisionManager, RevisionPersistence, SQLiteRevisionSnapshotPersistence}; +use flowy_revision::{ + RevisionManager, RevisionPersistence, RevisionPersistenceConfiguration, SQLiteRevisionSnapshotPersistence, +}; use lib_infra::future::AFFuture; use std::sync::Arc; @@ -253,7 +255,8 @@ pub async fn make_grid_view_rev_manager( let pool = user.db_pool()?; let disk_cache = SQLiteGridViewRevisionPersistence::new(&user_id, pool.clone()); - let rev_persistence = RevisionPersistence::new(&user_id, view_id, disk_cache); + let configuration = RevisionPersistenceConfiguration::default(); + let rev_persistence = RevisionPersistence::new(&user_id, view_id, disk_cache, configuration); let rev_compactor = GridViewRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(view_id, pool); diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs index 31cb05bb3a..088954e79b 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_block_impl.rs @@ -219,7 +219,7 @@ impl std::default::Default for GridBlockRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridBlockRevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: GridBlockRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs index e1ab04854c..e3a4e25625 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_impl.rs @@ -217,7 +217,7 @@ impl std::default::Default for GridRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridRevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: GridRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs index d86451412c..9aad02113e 100644 --- a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs +++ b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_view_impl.rs @@ -219,7 +219,7 @@ impl std::default::Default for GridViewRevisionState { } } -fn mk_revision_record_from_table(user_id: &str, table: GridViewRevisionTable) -> SyncRecord { +fn mk_revision_record_from_table(_user_id: &str, table: GridViewRevisionTable) -> SyncRecord { let md5 = md5(&table.data); let revision = Revision::new( &table.object_id, diff --git a/frontend/rust-lib/flowy-revision/src/cache/memory.rs b/frontend/rust-lib/flowy-revision/src/cache/memory.rs index ad6795437c..8d83ebd4cb 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/memory.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/memory.rs @@ -88,6 +88,10 @@ impl RevisionMemoryCache { Ok(revs) } + pub(crate) fn number_of_sync_records(&self) -> usize { + self.revs_map.len() + } + pub(crate) async fn reset_with_revisions(&self, revision_records: Vec) { self.revs_map.clear(); if let Some(handler) = self.defer_save.write().await.take() { diff --git a/frontend/rust-lib/flowy-revision/src/cache/reset.rs b/frontend/rust-lib/flowy-revision/src/cache/reset.rs index c4dc8ae970..4c4c223818 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/reset.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/reset.rs @@ -1,5 +1,5 @@ use crate::disk::{RevisionDiskCache, SyncRecord}; -use crate::{RevisionLoader, RevisionPersistence}; +use crate::{RevisionLoader, RevisionPersistence, RevisionPersistenceConfiguration}; use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_sync::entities::revision::Revision; @@ -60,10 +60,12 @@ where } async fn reset_object(&self) -> FlowyResult<()> { + let configuration = RevisionPersistenceConfiguration::new(2); let rev_persistence = Arc::new(RevisionPersistence::from_disk_cache( &self.user_id, self.target.target_id(), self.disk_cache.clone(), + configuration, )); let (revisions, _) = RevisionLoader { object_id: self.target.target_id().to_owned(), diff --git a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs index 2700cc4f1f..f3e1f1548c 100644 --- a/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs +++ b/frontend/rust-lib/flowy-revision/src/conflict_resolve.rs @@ -151,7 +151,7 @@ where } fn make_client_and_server_revision( - user_id: &str, + _user_id: &str, rev_manager: &Arc>, client_operations: Operations, server_operations: Option, diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index 32affc3200..a978c32760 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -42,13 +42,8 @@ pub trait RevisionObjectSerializer: Send + Sync { /// `RevisionCompress` is used to compress multiple revisions into one revision /// -pub trait RevisionCompress: Send + Sync { - fn compress_revisions( - &self, - user_id: &str, - object_id: &str, - mut revisions: Vec, - ) -> FlowyResult { +pub trait RevisionMergeable: Send + Sync { + fn merge_revisions(&self, _user_id: &str, object_id: &str, mut revisions: Vec) -> FlowyResult { if revisions.is_empty() { return Err(FlowyError::internal().context("Can't compact the empty revisions")); } @@ -69,18 +64,6 @@ pub trait RevisionCompress: Send + Sync { fn combine_revisions(&self, revisions: Vec) -> FlowyResult; } -pub struct RevisionConfiguration { - merge_when_excess_number_of_version: i64, -} - -impl std::default::Default for RevisionConfiguration { - fn default() -> Self { - Self { - merge_when_excess_number_of_version: 100, - } - } -} - pub struct RevisionManager { pub object_id: String, user_id: String, @@ -88,10 +71,9 @@ pub struct RevisionManager { rev_persistence: Arc>, #[allow(dead_code)] rev_snapshot: Arc, - rev_compress: Arc, + rev_compress: Arc, #[cfg(feature = "flowy_unit_test")] rev_ack_notifier: tokio::sync::broadcast::Sender, - // configuration: RevisionConfiguration, } impl RevisionManager { @@ -104,7 +86,7 @@ impl RevisionManager { ) -> Self where SP: 'static + RevisionSnapshotDiskCache, - C: 'static + RevisionCompress, + C: 'static + RevisionMergeable, { let rev_id_counter = RevIdCounter::new(0); let rev_compress = Arc::new(rev_compress); @@ -213,6 +195,10 @@ impl RevisionManager { (cur, next) } + pub fn number_of_sync_revisions(&self) -> usize { + self.rev_persistence.number_of_sync_records() + } + pub async fn get_revisions_in_range(&self, range: RevisionRange) -> Result, FlowyError> { let revisions = self.rev_persistence.revisions_in_range(&range).await?; Ok(revisions) diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index 57988ca90e..ac40390771 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -4,7 +4,7 @@ use crate::cache::{ }; use crate::disk::{RevisionState, SyncRecord}; use crate::memory::RevisionMemoryCache; -use crate::RevisionCompress; +use crate::RevisionMergeable; use flowy_error::{internal_error, FlowyError, FlowyResult}; use flowy_sync::entities::revision::{Revision, RevisionRange}; use std::collections::VecDeque; @@ -14,30 +14,58 @@ use tokio::task::spawn_blocking; pub const REVISION_WRITE_INTERVAL_IN_MILLIS: u64 = 600; +pub struct RevisionPersistenceConfiguration { + merge_threshold: usize, +} + +impl RevisionPersistenceConfiguration { + pub fn new(merge_threshold: usize) -> Self { + debug_assert!(merge_threshold > 1); + if merge_threshold > 1 { + Self { merge_threshold } + } else { + Self { merge_threshold: 2 } + } + } +} + +impl std::default::Default for RevisionPersistenceConfiguration { + fn default() -> Self { + Self { merge_threshold: 2 } + } +} + pub struct RevisionPersistence { user_id: String, object_id: String, disk_cache: Arc>, memory_cache: Arc, sync_seq: RwLock, + configuration: RevisionPersistenceConfiguration, } impl RevisionPersistence where Connection: 'static, { - pub fn new(user_id: &str, object_id: &str, disk_cache: C) -> RevisionPersistence + pub fn new( + user_id: &str, + object_id: &str, + disk_cache: C, + configuration: RevisionPersistenceConfiguration, + ) -> RevisionPersistence where C: 'static + RevisionDiskCache, { let disk_cache = Arc::new(disk_cache) as Arc>; - Self::from_disk_cache(user_id, object_id, disk_cache) + Self::from_disk_cache(user_id, object_id, disk_cache, configuration) } pub fn from_disk_cache( user_id: &str, object_id: &str, disk_cache: Arc>, + configuration: RevisionPersistenceConfiguration, ) -> RevisionPersistence { let object_id = object_id.to_owned(); let user_id = user_id.to_owned(); @@ -49,6 +77,7 @@ where disk_cache, memory_cache, sync_seq, + configuration, } } @@ -64,7 +93,7 @@ where pub(crate) async fn sync_revision(&self, revision: &Revision) -> FlowyResult<()> { tracing::Span::current().record("rev_id", &revision.rev_id); self.add(revision.clone(), RevisionState::Sync, false).await?; - self.sync_seq.write().await.add(revision.rev_id)?; + self.sync_seq.write().await.dry_push(revision.rev_id)?; Ok(()) } @@ -72,44 +101,39 @@ where #[tracing::instrument(level = "trace", skip_all, fields(rev_id, compact_range, object_id=%self.object_id), err)] pub(crate) async fn add_sync_revision<'a>( &'a self, - revision: &'a Revision, - rev_compress: &Arc, + new_revision: &'a Revision, + rev_compress: &Arc, ) -> FlowyResult { let mut sync_seq_write_guard = self.sync_seq.write().await; - let result = sync_seq_write_guard.compact(); - match result { - None => { - tracing::Span::current().record("rev_id", &revision.rev_id); - self.add(revision.clone(), RevisionState::Sync, true).await?; - sync_seq_write_guard.add(revision.rev_id)?; - Ok(revision.rev_id) - } - Some((range, mut compact_seq)) => { - tracing::Span::current().record("compact_range", &format!("{}", range).as_str()); - let mut revisions = self.revisions_in_range(&range).await?; - if range.to_rev_ids().len() != revisions.len() { - debug_assert_eq!(range.to_rev_ids().len(), revisions.len()); - } + if sync_seq_write_guard.step > self.configuration.merge_threshold { + let compact_seq = sync_seq_write_guard.compact(); + let range = RevisionRange { + start: *compact_seq.front().unwrap(), + end: *compact_seq.back().unwrap(), + }; - // append the new revision - revisions.push(revision.clone()); + tracing::Span::current().record("compact_range", &format!("{}", range).as_str()); + let mut revisions = self.revisions_in_range(&range).await?; + debug_assert_eq!(range.len() as usize, revisions.len()); + // append the new revision + revisions.push(new_revision.clone()); - // compact multiple revisions into one - let compact_revision = rev_compress.compress_revisions(&self.user_id, &self.object_id, revisions)?; - let rev_id = compact_revision.rev_id; - tracing::Span::current().record("rev_id", &rev_id); + // compact multiple revisions into one + let compact_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?; + let rev_id = compact_revision.rev_id; + tracing::Span::current().record("rev_id", &rev_id); - // insert new revision - compact_seq.push_back(rev_id); + // insert new revision + let _ = sync_seq_write_guard.dry_push(rev_id)?; - // replace the revisions in range with compact revision - self.compact(&range, compact_revision).await?; - // - debug_assert_eq!(compact_seq.len(), 2); - debug_assert_eq!(sync_seq_write_guard.len(), compact_seq.len()); - sync_seq_write_guard.reset(compact_seq); - Ok(rev_id) - } + // replace the revisions in range with compact revision + self.compact(&range, compact_revision).await?; + Ok(rev_id) + } else { + tracing::Span::current().record("rev_id", &new_revision.rev_id); + self.add(new_revision.clone(), RevisionState::Sync, true).await?; + sync_seq_write_guard.push(new_revision.rev_id)?; + Ok(new_revision.rev_id) } } @@ -132,6 +156,10 @@ where self.sync_seq.read().await.next_rev_id() } + pub(crate) fn number_of_sync_records(&self) -> usize { + self.memory_cache.number_of_sync_records() + } + /// The cache gets reset while it conflicts with the remote revisions. #[tracing::instrument(level = "trace", skip(self, revisions), err)] pub(crate) async fn reset(&self, revisions: Vec) -> FlowyResult<()> { @@ -257,27 +285,42 @@ impl RevisionMemoryCacheDelegate for Arc); +struct DeferSyncSequence { + rev_ids: VecDeque, + start: Option, + step: usize, +} + impl DeferSyncSequence { fn new() -> Self { DeferSyncSequence::default() } - fn add(&mut self, new_rev_id: i64) -> FlowyResult<()> { + fn push(&mut self, new_rev_id: i64) -> FlowyResult<()> { + let _ = self.dry_push(new_rev_id)?; + + self.step += 1; + if self.start.is_none() && !self.rev_ids.is_empty() { + self.start = Some(self.rev_ids.len() - 1); + } + Ok(()) + } + + fn dry_push(&mut self, new_rev_id: i64) -> FlowyResult<()> { // The last revision's rev_id must be greater than the new one. - if let Some(rev_id) = self.0.back() { + if let Some(rev_id) = self.rev_ids.back() { if *rev_id >= new_rev_id { return Err( FlowyError::internal().context(format!("The new revision's id must be greater than {}", rev_id)) ); } } - self.0.push_back(new_rev_id); + self.rev_ids.push_back(new_rev_id); Ok(()) } fn ack(&mut self, rev_id: &i64) -> FlowyResult<()> { - let cur_rev_id = self.0.front().cloned(); + let cur_rev_id = self.rev_ids.front().cloned(); if let Some(pop_rev_id) = cur_rev_id { if &pop_rev_id != rev_id { let desc = format!( @@ -286,38 +329,38 @@ impl DeferSyncSequence { ); return Err(FlowyError::internal().context(desc)); } - let _ = self.0.pop_front(); + let _ = self.rev_ids.pop_front(); } Ok(()) } fn next_rev_id(&self) -> Option { - self.0.front().cloned() - } - - fn reset(&mut self, new_seq: VecDeque) { - self.0 = new_seq; + self.rev_ids.front().cloned() } fn clear(&mut self) { - self.0.clear(); - } - - fn len(&self) -> usize { - self.0.len() + self.start = None; + self.step = 0; + self.rev_ids.clear(); } // Compact the rev_ids into one except the current synchronizing rev_id. - fn compact(&self) -> Option<(RevisionRange, VecDeque)> { - // Make sure there are two rev_id going to sync. No need to compact if there is only - // one rev_id in queue. - self.next_rev_id()?; + fn compact(&mut self) -> VecDeque { + if self.start.is_none() { + return VecDeque::default(); + } - let mut new_seq = self.0.clone(); - let mut drained = new_seq.drain(1..).collect::>(); + let start = self.start.unwrap(); + let compact_seq = self.rev_ids.split_off(start); + self.start = None; + self.step = 0; + compact_seq - let start = drained.pop_front()?; - let end = drained.pop_back().unwrap_or(start); - Some((RevisionRange { start, end }, new_seq)) + // let mut new_seq = self.rev_ids.clone(); + // let mut drained = new_seq.drain(1..).collect::>(); + // + // let start = drained.pop_front()?; + // let end = drained.pop_back().unwrap_or(start); + // Some((RevisionRange { start, end }, new_seq)) } } diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs index 4c3aa97ae0..88ee0bc0c9 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs @@ -20,7 +20,7 @@ async fn revision_sync_test() { #[tokio::test] async fn revision_sync_multiple_revisions() { - let test = RevisionTest::new().await; + let test = RevisionTest::new_with_configuration(2).await; let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { @@ -49,21 +49,20 @@ async fn revision_sync_multiple_revisions() { } #[tokio::test] -async fn revision_compress_two_revisions_test() { - let test = RevisionTest::new().await; +async fn revision_compress_three_revisions_test() { + let test = RevisionTest::new_with_configuration(2).await; let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { - content: "123".to_string(), + content: "1".to_string(), base_rev_id, rev_id: rev_id_1, }) .await; - // rev_id_2 will be merged with rev_id_3 let (base_rev_id, rev_id_2) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { - content: "456".to_string(), + content: "2".to_string(), base_rev_id, rev_id: rev_id_2, }) @@ -71,36 +70,129 @@ async fn revision_compress_two_revisions_test() { let (base_rev_id, rev_id_3) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { - content: "789".to_string(), + content: "3".to_string(), base_rev_id, rev_id: rev_id_3, }) .await; + let (base_rev_id, rev_id_4) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "4".to_string(), + base_rev_id, + rev_id: rev_id_4, + }) + .await; + + // rev_id_2,rev_id_3,rev_id4 will be merged into rev_id_1 test.run_scripts(vec![ Wait { milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, }, + AssertNumberOfSyncRevisions { num: 1 }, AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, - AckRevision { rev_id: rev_id_1 }, - AssertNextSyncRevisionId { rev_id: Some(rev_id_2) }, AssertNextSyncRevisionContent { - expected: "456789".to_string(), + expected: "1234".to_string(), }, + AckRevision { rev_id: rev_id_1 }, + AssertNextSyncRevisionId { rev_id: None }, ]) .await; } #[tokio::test] -async fn revision_compress_multiple_revisions_test() { - let test = RevisionTest::new().await; - let mut expected = "".to_owned(); +async fn revision_compress_three_revisions_test2() { + let test = RevisionTest::new_with_configuration(2).await; + let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); - for i in 0..100 { + test.run_script(AddLocalRevision { + content: "1".to_string(), + base_rev_id, + rev_id: rev_id_1, + }) + .await; + + let (base_rev_id, rev_id_2) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "2".to_string(), + base_rev_id, + rev_id: rev_id_2, + }) + .await; + + let (base_rev_id, rev_id_3) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "3".to_string(), + base_rev_id, + rev_id: rev_id_3, + }) + .await; + + let (base_rev_id, rev_id_4) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "4".to_string(), + base_rev_id, + rev_id: rev_id_4, + }) + .await; + + let (base_rev_id, rev_id_a) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "a".to_string(), + base_rev_id, + rev_id: rev_id_a, + }) + .await; + + let (base_rev_id, rev_id_b) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "b".to_string(), + base_rev_id, + rev_id: rev_id_b, + }) + .await; + + let (base_rev_id, rev_id_c) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "c".to_string(), + base_rev_id, + rev_id: rev_id_c, + }) + .await; + + let (base_rev_id, rev_id_d) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "d".to_string(), + base_rev_id, + rev_id: rev_id_d, + }) + .await; + + test.run_scripts(vec![ + // Wait { + // milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, + // }, + AssertNumberOfSyncRevisions { num: 2 }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, + AssertNextSyncRevisionContent { + expected: "1234".to_string(), + }, + AckRevision { rev_id: rev_id_1 }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_a) }, + AssertNextSyncRevisionContent { + expected: "abcd".to_string(), + }, + AckRevision { rev_id: rev_id_a }, + AssertNextSyncRevisionId { rev_id: None }, + ]) + .await; +} + +#[tokio::test] +async fn revision_merge_per_5_revision_test() { + let test = RevisionTest::new_with_configuration(4).await; + for i in 0..20 { let content = format!("{}", i); - if i != 0 { - expected.push_str(&content); - } let (base_rev_id, rev_id) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { content, @@ -110,14 +202,5 @@ async fn revision_compress_multiple_revisions_test() { .await; } - test.run_scripts(vec![ - Wait { - milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, - }, - AssertNextSyncRevisionId { rev_id: Some(1) }, - AckRevision { rev_id: 1 }, - AssertNextSyncRevisionId { rev_id: Some(2) }, - AssertNextSyncRevisionContent { expected }, - ]) - .await; + test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 5 }]).await; } diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs index 8aeba313cf..dce38a0c09 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -2,7 +2,8 @@ use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, SyncRecord}; use flowy_revision::{ - RevisionCompress, RevisionManager, RevisionPersistence, RevisionSnapshotDiskCache, RevisionSnapshotInfo, + RevisionManager, RevisionMergeable, RevisionPersistence, RevisionPersistenceConfiguration, + RevisionSnapshotDiskCache, RevisionSnapshotInfo, }; use flowy_sync::entities::revision::{Revision, RevisionRange}; use flowy_sync::util::md5; @@ -11,7 +12,6 @@ use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::time::Duration; -use tokio::time::interval; pub enum RevisionScript { AddLocalRevision { @@ -25,14 +25,15 @@ pub enum RevisionScript { AssertNextSyncRevisionId { rev_id: Option, }, + AssertNumberOfSyncRevisions { + num: usize, + }, AssertNextSyncRevisionContent { expected: String, }, Wait { milliseconds: u64, }, - - AssertNextSyncRevision(Option), } pub struct RevisionTest { @@ -41,9 +42,14 @@ pub struct RevisionTest { impl RevisionTest { pub async fn new() -> Self { + Self::new_with_configuration(2).await + } + + pub async fn new_with_configuration(merge_when_excess_number_of_version: i64) -> Self { let user_id = nanoid!(10); let object_id = nanoid!(6); - let persistence = RevisionPersistence::new(&user_id, &object_id, RevisionDiskCacheMock::new()); + let configuration = RevisionPersistenceConfiguration::new(merge_when_excess_number_of_version as usize); + let persistence = RevisionPersistence::new(&user_id, &object_id, RevisionDiskCacheMock::new(), configuration); let compress = RevisionCompressMock {}; let snapshot = RevisionSnapshotMock {}; let rev_manager = RevisionManager::new(&user_id, &object_id, persistence, compress, snapshot); @@ -51,6 +57,7 @@ impl RevisionTest { rev_manager: Arc::new(rev_manager), } } + pub async fn run_scripts(&self, scripts: Vec) { for script in scripts { self.run_script(script).await; @@ -87,6 +94,9 @@ impl RevisionTest { RevisionScript::AssertNextSyncRevisionId { rev_id } => { assert_eq!(self.rev_manager.next_sync_rev_id().await, rev_id) } + RevisionScript::AssertNumberOfSyncRevisions { num } => { + assert_eq!(self.rev_manager.number_of_sync_revisions(), num) + } RevisionScript::AssertNextSyncRevisionContent { expected } => { // let rev_id = self.rev_manager.next_sync_rev_id().await.unwrap(); @@ -95,14 +105,8 @@ impl RevisionTest { assert_eq!(object.content, expected); } RevisionScript::Wait { milliseconds } => { - // let mut interval = interval(Duration::from_millis(milliseconds)); - // interval.tick().await; tokio::time::sleep(Duration::from_millis(milliseconds)).await; } - RevisionScript::AssertNextSyncRevision(expected) => { - let next_revision = self.rev_manager.next_sync_revision().await.unwrap(); - assert_eq!(next_revision, expected); - } } } } @@ -133,16 +137,16 @@ impl RevisionDiskCache for RevisionDiskCacheMock { fn read_revision_records( &self, - object_id: &str, - rev_ids: Option>, + _object_id: &str, + _rev_ids: Option>, ) -> Result, Self::Error> { todo!() } fn read_revision_records_with_range( &self, - object_id: &str, - range: &RevisionRange, + _object_id: &str, + _range: &RevisionRange, ) -> Result, Self::Error> { todo!() } @@ -161,7 +165,7 @@ impl RevisionDiskCache for RevisionDiskCacheMock { Ok(()) } - fn delete_revision_records(&self, object_id: &str, rev_ids: Option>) -> Result<(), Self::Error> { + fn delete_revision_records(&self, _object_id: &str, rev_ids: Option>) -> Result<(), Self::Error> { match rev_ids { None => {} Some(rev_ids) => { @@ -182,9 +186,9 @@ impl RevisionDiskCache for RevisionDiskCacheMock { fn delete_and_insert_records( &self, - object_id: &str, - deleted_rev_ids: Option>, - inserted_records: Vec, + _object_id: &str, + _deleted_rev_ids: Option>, + _inserted_records: Vec, ) -> Result<(), Self::Error> { todo!() } @@ -195,18 +199,18 @@ pub struct RevisionConnectionMock {} pub struct RevisionSnapshotMock {} impl RevisionSnapshotDiskCache for RevisionSnapshotMock { - fn write_snapshot(&self, object_id: &str, rev_id: i64, data: Vec) -> FlowyResult<()> { + fn write_snapshot(&self, _object_id: &str, _rev_id: i64, _data: Vec) -> FlowyResult<()> { todo!() } - fn read_snapshot(&self, object_id: &str, rev_id: i64) -> FlowyResult { + fn read_snapshot(&self, _object_id: &str, _rev_id: i64) -> FlowyResult { todo!() } } pub struct RevisionCompressMock {} -impl RevisionCompress for RevisionCompressMock { +impl RevisionMergeable for RevisionCompressMock { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { let mut object = RevisionObjectMock::new(""); for revision in revisions { diff --git a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs index 33e588f99f..1fa6804177 100644 --- a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs +++ b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs @@ -144,7 +144,7 @@ struct DocumentViewDataProcessor(Arc); impl ViewDataProcessor for DocumentViewDataProcessor { fn create_view( &self, - user_id: &str, + _user_id: &str, view_id: &str, layout: ViewLayoutTypePB, view_data: Bytes, @@ -188,7 +188,7 @@ impl ViewDataProcessor for DocumentViewDataProcessor { _data_format: ViewDataFormatPB, ) -> FutureResult { debug_assert_eq!(layout, ViewLayoutTypePB::Document); - let user_id = user_id.to_string(); + let _user_id = user_id.to_string(); let view_id = view_id.to_string(); let manager = self.0.clone(); let document_content = self.0.initial_document_content(); @@ -220,7 +220,7 @@ struct GridViewDataProcessor(Arc); impl ViewDataProcessor for GridViewDataProcessor { fn create_view( &self, - user_id: &str, + _user_id: &str, view_id: &str, _layout: ViewLayoutTypePB, delta_data: Bytes, diff --git a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs index ebf578cc3c..5a95510f7f 100644 --- a/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/block_revision_pad.rs @@ -256,7 +256,7 @@ pub fn make_grid_block_operations(block_rev: &GridBlockRevision) -> GridBlockOpe GridBlockOperationsBuilder::new().insert(&json).build() } -pub fn make_grid_block_revisions(user_id: &str, grid_block_meta_data: &GridBlockRevision) -> RepeatedRevision { +pub fn make_grid_block_revisions(_user_id: &str, grid_block_meta_data: &GridBlockRevision) -> RepeatedRevision { let operations = make_grid_block_operations(grid_block_meta_data); let bytes = operations.json_bytes(); let revision = Revision::initial_revision(&grid_block_meta_data.block_id, bytes); diff --git a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs index 5f3d7860ae..2bcc1377c2 100644 --- a/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs +++ b/shared-lib/flowy-sync/src/client_grid/grid_revision_pad.rs @@ -409,7 +409,7 @@ pub fn make_grid_operations(grid_rev: &GridRevision) -> GridOperations { GridOperationsBuilder::new().insert(&json).build() } -pub fn make_grid_revisions(user_id: &str, grid_rev: &GridRevision) -> RepeatedRevision { +pub fn make_grid_revisions(_user_id: &str, grid_rev: &GridRevision) -> RepeatedRevision { let operations = make_grid_operations(grid_rev); let bytes = operations.json_bytes(); let revision = Revision::initial_revision(&grid_rev.grid_id, bytes); diff --git a/shared-lib/flowy-sync/src/entities/revision.rs b/shared-lib/flowy-sync/src/entities/revision.rs index e8b7e42d88..ede5a6eca2 100644 --- a/shared-lib/flowy-sync/src/entities/revision.rs +++ b/shared-lib/flowy-sync/src/entities/revision.rs @@ -178,10 +178,10 @@ impl std::fmt::Display for RevisionRange { } impl RevisionRange { - pub fn len(&self) -> i64 { + pub fn len(&self) -> u64 { debug_assert!(self.end >= self.start); if self.end >= self.start { - self.end - self.start + 1 + (self.end - self.start + 1) as u64 } else { 0 } From e729c0a81f74b7056042a2947467a182daaa0e0b Mon Sep 17 00:00:00 2001 From: nathan Date: Mon, 7 Nov 2022 10:09:05 +0800 Subject: [PATCH 5/9] chore: add documentation --- .../flowy-revision/src/rev_persistence.rs | 26 ++++++------ .../revision_test/local_revision_test.rs | 41 ++++++++++++++++++- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index ac40390771..62d4a0e408 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -104,9 +104,14 @@ where new_revision: &'a Revision, rev_compress: &Arc, ) -> FlowyResult { - let mut sync_seq_write_guard = self.sync_seq.write().await; - if sync_seq_write_guard.step > self.configuration.merge_threshold { - let compact_seq = sync_seq_write_guard.compact(); + let mut sync_seq = self.sync_seq.write().await; + let step = sync_seq.step; + + // Before the new_revision pushed into the sync_seq, we check if the current `step` of the + // sync_seq is less equal or greater than the merge threshold. If yes, it's need to merged + // with the new_revision into one revision. + if step >= self.configuration.merge_threshold - 1 { + let compact_seq = sync_seq.compact(); let range = RevisionRange { start: *compact_seq.front().unwrap(), end: *compact_seq.back().unwrap(), @@ -119,20 +124,18 @@ where revisions.push(new_revision.clone()); // compact multiple revisions into one - let compact_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?; - let rev_id = compact_revision.rev_id; - tracing::Span::current().record("rev_id", &rev_id); - - // insert new revision - let _ = sync_seq_write_guard.dry_push(rev_id)?; + let merged_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?; + let rev_id = merged_revision.rev_id; + tracing::Span::current().record("rev_id", &merged_revision.rev_id); + let _ = sync_seq.dry_push(merged_revision.rev_id)?; // replace the revisions in range with compact revision - self.compact(&range, compact_revision).await?; + self.compact(&range, merged_revision).await?; Ok(rev_id) } else { tracing::Span::current().record("rev_id", &new_revision.rev_id); self.add(new_revision.clone(), RevisionState::Sync, true).await?; - sync_seq_write_guard.push(new_revision.rev_id)?; + sync_seq.push(new_revision.rev_id)?; Ok(new_revision.rev_id) } } @@ -201,7 +204,6 @@ where let _ = self .disk_cache .delete_revision_records(&self.object_id, Some(rev_ids))?; - self.add(new_revision, RevisionState::Sync, true).await?; Ok(()) } diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs index 88ee0bc0c9..b91d32386d 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs @@ -190,7 +190,7 @@ async fn revision_compress_three_revisions_test2() { #[tokio::test] async fn revision_merge_per_5_revision_test() { - let test = RevisionTest::new_with_configuration(4).await; + let test = RevisionTest::new_with_configuration(5).await; for i in 0..20 { let content = format!("{}", i); let (base_rev_id, rev_id) = test.next_rev_id_pair(); @@ -202,5 +202,42 @@ async fn revision_merge_per_5_revision_test() { .await; } - test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 5 }]).await; + test.run_scripts(vec![ + AssertNumberOfSyncRevisions { num: 4 }, + AssertNextSyncRevisionContent { + expected: "01234".to_string(), + }, + AckRevision { rev_id: 1 }, + AssertNextSyncRevisionContent { + expected: "56789".to_string(), + }, + AckRevision { rev_id: 2 }, + AssertNextSyncRevisionContent { + expected: "1011121314".to_string(), + }, + AckRevision { rev_id: 3 }, + AssertNextSyncRevisionContent { + expected: "1516171819".to_string(), + }, + AckRevision { rev_id: 4 }, + AssertNextSyncRevisionId { rev_id: None }, + ]) + .await; +} + +#[tokio::test] +async fn revision_merge_per_100_revision_test() { + let test = RevisionTest::new_with_configuration(100).await; + for i in 0..1000 { + let content = format!("{}", i); + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content, + base_rev_id, + rev_id, + }) + .await; + } + + test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 10 }]).await; } From b3b24d0cc058c6ca151f027bdd95978a9e5e8cad Mon Sep 17 00:00:00 2001 From: appflowy Date: Mon, 7 Nov 2022 17:30:24 +0800 Subject: [PATCH 6/9] chore: calculate the compact length after receiving ack --- frontend/rust-lib/Cargo.lock | 1 + .../flowy-document/src/editor/editor.rs | 4 +- .../flowy-document/src/old_editor/editor.rs | 2 +- .../src/services/folder_editor.rs | 4 +- .../flowy-grid/src/services/block_editor.rs | 2 +- .../flowy-grid/src/services/grid_editor.rs | 2 +- .../src/services/grid_view_editor.rs | 2 +- frontend/rust-lib/flowy-revision/Cargo.toml | 1 + .../flowy-revision/src/rev_manager.rs | 15 +- .../flowy-revision/src/rev_persistence.rs | 102 +++++++---- .../revision_test/local_revision_test.rs | 126 ++++++++++--- .../flowy-revision/tests/revision_test/mod.rs | 1 + .../tests/revision_test/revision_disk_test.rs | 103 +++++++++++ .../tests/revision_test/script.rs | 173 +++++++++++++++--- frontend/rust-lib/flowy-sdk/src/lib.rs | 2 +- 15 files changed, 446 insertions(+), 94 deletions(-) create mode 100644 frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index a8871e6302..566a6169e2 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -1072,6 +1072,7 @@ dependencies = [ "bytes", "dashmap", "flowy-error", + "flowy-revision", "flowy-sync", "futures-util", "lib-infra", diff --git a/frontend/rust-lib/flowy-document/src/editor/editor.rs b/frontend/rust-lib/flowy-document/src/editor/editor.rs index 5270932a1b..eee8eaa6de 100644 --- a/frontend/rust-lib/flowy-document/src/editor/editor.rs +++ b/frontend/rust-lib/flowy-document/src/editor/editor.rs @@ -29,7 +29,9 @@ impl AppFlowyDocumentEditor { mut rev_manager: RevisionManager>, cloud_service: Arc, ) -> FlowyResult> { - let document = rev_manager.load::(Some(cloud_service)).await?; + let document = rev_manager + .initialize::(Some(cloud_service)) + .await?; let rev_manager = Arc::new(rev_manager); let command_sender = spawn_edit_queue(user, rev_manager.clone(), document); let doc_id = doc_id.to_string(); diff --git a/frontend/rust-lib/flowy-document/src/old_editor/editor.rs b/frontend/rust-lib/flowy-document/src/old_editor/editor.rs index 35de2de668..e48842351d 100644 --- a/frontend/rust-lib/flowy-document/src/old_editor/editor.rs +++ b/frontend/rust-lib/flowy-document/src/old_editor/editor.rs @@ -45,7 +45,7 @@ impl DeltaDocumentEditor { cloud_service: Arc, ) -> FlowyResult> { let document = rev_manager - .load::(Some(cloud_service)) + .initialize::(Some(cloud_service)) .await?; let operations = DeltaTextOperations::from_bytes(&document.content)?; let rev_manager = Arc::new(rev_manager); diff --git a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs index 93ec061565..2ad6a58f55 100644 --- a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs +++ b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs @@ -38,7 +38,9 @@ impl FolderEditor { let cloud = Arc::new(FolderRevisionCloudService { token: token.to_string(), }); - let folder = Arc::new(RwLock::new(rev_manager.load::(Some(cloud)).await?)); + let folder = Arc::new(RwLock::new( + rev_manager.initialize::(Some(cloud)).await?, + )); let rev_manager = Arc::new(rev_manager); #[cfg(feature = "sync")] diff --git a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs index 2a8d01cb54..9ee6278bd6 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_editor.rs @@ -34,7 +34,7 @@ impl GridBlockRevisionEditor { let cloud = Arc::new(GridBlockRevisionCloudService { token: token.to_owned(), }); - let block_revision_pad = rev_manager.load::(Some(cloud)).await?; + let block_revision_pad = rev_manager.initialize::(Some(cloud)).await?; let pad = Arc::new(RwLock::new(block_revision_pad)); let rev_manager = Arc::new(rev_manager); let user_id = user_id.to_owned(); diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs index 20e8cf34cb..8598652abc 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs @@ -60,7 +60,7 @@ impl GridRevisionEditor { ) -> FlowyResult> { let token = user.token()?; let cloud = Arc::new(GridRevisionCloudService { token }); - let grid_pad = rev_manager.load::(Some(cloud)).await?; + let grid_pad = rev_manager.initialize::(Some(cloud)).await?; let rev_manager = Arc::new(rev_manager); let grid_pad = Arc::new(RwLock::new(grid_pad)); diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs index 4be9043861..4e0f707708 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_editor.rs @@ -55,7 +55,7 @@ impl GridViewRevisionEditor { let cloud = Arc::new(GridViewRevisionCloudService { token: token.to_owned(), }); - let view_revision_pad = rev_manager.load::(Some(cloud)).await?; + let view_revision_pad = rev_manager.initialize::(Some(cloud)).await?; let pad = Arc::new(RwLock::new(view_revision_pad)); let rev_manager = Arc::new(rev_manager); let group_controller = new_group_controller( diff --git a/frontend/rust-lib/flowy-revision/Cargo.toml b/frontend/rust-lib/flowy-revision/Cargo.toml index 54f1902afa..8e6800e823 100644 --- a/frontend/rust-lib/flowy-revision/Cargo.toml +++ b/frontend/rust-lib/flowy-revision/Cargo.toml @@ -23,6 +23,7 @@ serde_json = {version = "1.0"} [dev-dependencies] nanoid = "0.4.0" +flowy-revision = {path = ".", features = ["flowy_unit_test"]} serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } parking_lot = "0.11" diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index a978c32760..7033894ca5 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -108,7 +108,7 @@ impl RevisionManager { } #[tracing::instrument(level = "debug", skip_all, fields(object_id) err)] - pub async fn load(&mut self, cloud: Option>) -> FlowyResult + pub async fn initialize(&mut self, cloud: Option>) -> FlowyResult where B: RevisionObjectDeserializer, { @@ -199,6 +199,10 @@ impl RevisionManager { self.rev_persistence.number_of_sync_records() } + pub fn number_of_revisions_in_disk(&self) -> usize { + self.rev_persistence.number_of_records_in_disk() + } + pub async fn get_revisions_in_range(&self, range: RevisionRange) -> Result, FlowyError> { let revisions = self.rev_persistence.revisions_in_range(&range).await?; Ok(revisions) @@ -230,13 +234,16 @@ impl WSDataProviderDataSource for Arc RevisionManager { +impl RevisionManager { pub async fn revision_cache(&self) -> Arc> { self.rev_persistence.clone() } pub fn ack_notify(&self) -> tokio::sync::broadcast::Receiver { self.rev_ack_notifier.subscribe() } + pub fn get_all_revision_records(&self) -> FlowyResult> { + self.rev_persistence.load_all_records(&self.object_id) + } } pub struct RevisionLoader { @@ -248,7 +255,7 @@ pub struct RevisionLoader { impl RevisionLoader { pub async fn load(&self) -> Result<(Vec, i64), FlowyError> { - let records = self.rev_persistence.batch_get(&self.object_id)?; + let records = self.rev_persistence.load_all_records(&self.object_id)?; let revisions: Vec; let mut rev_id = 0; if records.is_empty() && self.cloud.is_some() { @@ -282,7 +289,7 @@ impl RevisionLoader { } pub async fn load_revisions(&self) -> Result, FlowyError> { - let records = self.rev_persistence.batch_get(&self.object_id)?; + let records = self.rev_persistence.load_all_records(&self.object_id)?; let revisions = records.into_iter().map(|record| record.revision).collect::<_>(); Ok(revisions) } diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index 62d4a0e408..c74cb4e21e 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -14,6 +14,7 @@ use tokio::task::spawn_blocking; pub const REVISION_WRITE_INTERVAL_IN_MILLIS: u64 = 600; +#[derive(Clone)] pub struct RevisionPersistenceConfiguration { merge_threshold: usize, } @@ -24,14 +25,14 @@ impl RevisionPersistenceConfiguration { if merge_threshold > 1 { Self { merge_threshold } } else { - Self { merge_threshold: 2 } + Self { merge_threshold: 100 } } } } impl std::default::Default for RevisionPersistenceConfiguration { fn default() -> Self { - Self { merge_threshold: 2 } + Self { merge_threshold: 100 } } } @@ -93,7 +94,7 @@ where pub(crate) async fn sync_revision(&self, revision: &Revision) -> FlowyResult<()> { tracing::Span::current().record("rev_id", &revision.rev_id); self.add(revision.clone(), RevisionState::Sync, false).await?; - self.sync_seq.write().await.dry_push(revision.rev_id)?; + self.sync_seq.write().await.recv(revision.rev_id)?; Ok(()) } @@ -105,13 +106,17 @@ where rev_compress: &Arc, ) -> FlowyResult { let mut sync_seq = self.sync_seq.write().await; - let step = sync_seq.step; + let compact_length = sync_seq.compact_length; - // Before the new_revision pushed into the sync_seq, we check if the current `step` of the - // sync_seq is less equal or greater than the merge threshold. If yes, it's need to merged + // Before the new_revision is pushed into the sync_seq, we check if the current `step` of the + // sync_seq is less equal to or greater than the merge threshold. If yes, it's needs to merged // with the new_revision into one revision. - if step >= self.configuration.merge_threshold - 1 { - let compact_seq = sync_seq.compact(); + let mut compact_seq = VecDeque::default(); + // tracing::info!("{}", compact_seq) + if compact_length >= self.configuration.merge_threshold - 1 { + compact_seq.extend(sync_seq.compact()); + } + if !compact_seq.is_empty() { let range = RevisionRange { start: *compact_seq.front().unwrap(), end: *compact_seq.back().unwrap(), @@ -127,7 +132,7 @@ where let merged_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?; let rev_id = merged_revision.rev_id; tracing::Span::current().record("rev_id", &merged_revision.rev_id); - let _ = sync_seq.dry_push(merged_revision.rev_id)?; + let _ = sync_seq.recv(merged_revision.rev_id)?; // replace the revisions in range with compact revision self.compact(&range, merged_revision).await?; @@ -135,7 +140,7 @@ where } else { tracing::Span::current().record("rev_id", &new_revision.rev_id); self.add(new_revision.clone(), RevisionState::Sync, true).await?; - sync_seq.push(new_revision.rev_id)?; + sync_seq.merge_recv(new_revision.rev_id)?; Ok(new_revision.rev_id) } } @@ -163,6 +168,16 @@ where self.memory_cache.number_of_sync_records() } + pub(crate) fn number_of_records_in_disk(&self) -> usize { + match self.disk_cache.read_revision_records(&self.object_id, None) { + Ok(records) => records.len(), + Err(e) => { + tracing::error!("Read revision records failed: {:?}", e); + 0 + } + } + } + /// The cache gets reset while it conflicts with the remote revisions. #[tracing::instrument(level = "trace", skip(self, revisions), err)] pub(crate) async fn reset(&self, revisions: Vec) -> FlowyResult<()> { @@ -228,8 +243,8 @@ where } } - pub fn batch_get(&self, doc_id: &str) -> FlowyResult> { - self.disk_cache.read_revision_records(doc_id, None) + pub fn load_all_records(&self, object_id: &str) -> FlowyResult> { + self.disk_cache.read_revision_records(object_id, None) } // Read the revision which rev_id >= range.start && rev_id <= range.end @@ -289,8 +304,8 @@ impl RevisionMemoryCacheDelegate for Arc, - start: Option, - step: usize, + compact_index: Option, + compact_length: usize, } impl DeferSyncSequence { @@ -298,17 +313,22 @@ impl DeferSyncSequence { DeferSyncSequence::default() } - fn push(&mut self, new_rev_id: i64) -> FlowyResult<()> { - let _ = self.dry_push(new_rev_id)?; + /// Pushes the new_rev_id to the end of the list and marks this new_rev_id is mergeable. + /// + /// When calling `compact` method, it will return a list of revision ids started from + /// the `compact_start_pos`, and ends with the `compact_length`. + fn merge_recv(&mut self, new_rev_id: i64) -> FlowyResult<()> { + let _ = self.recv(new_rev_id)?; - self.step += 1; - if self.start.is_none() && !self.rev_ids.is_empty() { - self.start = Some(self.rev_ids.len() - 1); + self.compact_length += 1; + if self.compact_index.is_none() && !self.rev_ids.is_empty() { + self.compact_index = Some(self.rev_ids.len() - 1); } Ok(()) } - fn dry_push(&mut self, new_rev_id: i64) -> FlowyResult<()> { + /// Pushes the new_rev_id to the end of the list. + fn recv(&mut self, new_rev_id: i64) -> FlowyResult<()> { // The last revision's rev_id must be greater than the new one. if let Some(rev_id) = self.rev_ids.back() { if *rev_id >= new_rev_id { @@ -321,6 +341,7 @@ impl DeferSyncSequence { Ok(()) } + /// Removes the rev_id from the list fn ack(&mut self, rev_id: &i64) -> FlowyResult<()> { let cur_rev_id = self.rev_ids.front().cloned(); if let Some(pop_rev_id) = cur_rev_id { @@ -331,7 +352,20 @@ impl DeferSyncSequence { ); return Err(FlowyError::internal().context(desc)); } - let _ = self.rev_ids.pop_front(); + + let mut compact_rev_id = None; + if let Some(compact_index) = self.compact_index { + compact_rev_id = self.rev_ids.get(compact_index).cloned(); + } + + let pop_rev_id = self.rev_ids.pop_front(); + if let (Some(compact_rev_id), Some(pop_rev_id)) = (compact_rev_id, pop_rev_id) { + if compact_rev_id <= pop_rev_id { + if self.compact_length > 0 { + self.compact_length -= 1; + } + } + } } Ok(()) } @@ -341,28 +375,22 @@ impl DeferSyncSequence { } fn clear(&mut self) { - self.start = None; - self.step = 0; + self.compact_index = None; + self.compact_length = 0; self.rev_ids.clear(); } // Compact the rev_ids into one except the current synchronizing rev_id. fn compact(&mut self) -> VecDeque { - if self.start.is_none() { - return VecDeque::default(); + let mut compact_seq = VecDeque::with_capacity(self.rev_ids.len()); + if let Some(start) = self.compact_index { + if start < self.rev_ids.len() { + let seq = self.rev_ids.split_off(start); + compact_seq.extend(seq); + } } - - let start = self.start.unwrap(); - let compact_seq = self.rev_ids.split_off(start); - self.start = None; - self.step = 0; + self.compact_index = None; + self.compact_length = 0; compact_seq - - // let mut new_seq = self.rev_ids.clone(); - // let mut drained = new_seq.drain(1..).collect::>(); - // - // let start = drained.pop_front()?; - // let end = drained.pop_back().unwrap_or(start); - // Some((RevisionRange { start, end }, new_seq)) } } diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs index b91d32386d..2e4d3119f5 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs @@ -19,37 +19,31 @@ async fn revision_sync_test() { } #[tokio::test] -async fn revision_sync_multiple_revisions() { +async fn revision_compress_2_revisions_with_2_threshold_test() { let test = RevisionTest::new_with_configuration(2).await; - let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); - test.run_script(AddLocalRevision { + test.run_script(AddLocalRevision2 { content: "123".to_string(), - base_rev_id, - rev_id: rev_id_1, + pair_rev_id: test.next_rev_id_pair(), }) .await; - let (base_rev_id, rev_id_2) = test.next_rev_id_pair(); - test.run_script(AddLocalRevision { + test.run_script(AddLocalRevision2 { content: "456".to_string(), - base_rev_id, - rev_id: rev_id_2, + pair_rev_id: test.next_rev_id_pair(), }) .await; test.run_scripts(vec![ - AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, - AckRevision { rev_id: rev_id_1 }, - AssertNextSyncRevisionId { rev_id: Some(rev_id_2) }, - AckRevision { rev_id: rev_id_2 }, + AssertNextSyncRevisionId { rev_id: Some(1) }, + AckRevision { rev_id: 1 }, AssertNextSyncRevisionId { rev_id: None }, ]) .await; } #[tokio::test] -async fn revision_compress_three_revisions_test() { +async fn revision_compress_4_revisions_with_threshold_2_test() { let test = RevisionTest::new_with_configuration(2).await; let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); @@ -86,23 +80,23 @@ async fn revision_compress_three_revisions_test() { // rev_id_2,rev_id_3,rev_id4 will be merged into rev_id_1 test.run_scripts(vec![ - Wait { - milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, - }, - AssertNumberOfSyncRevisions { num: 1 }, + AssertNumberOfSyncRevisions { num: 2 }, AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, AssertNextSyncRevisionContent { - expected: "1234".to_string(), + expected: "12".to_string(), }, AckRevision { rev_id: rev_id_1 }, - AssertNextSyncRevisionId { rev_id: None }, + AssertNextSyncRevisionId { rev_id: Some(rev_id_2) }, + AssertNextSyncRevisionContent { + expected: "34".to_string(), + }, ]) .await; } #[tokio::test] -async fn revision_compress_three_revisions_test2() { - let test = RevisionTest::new_with_configuration(2).await; +async fn revision_compress_8_revisions_with_threshold_4_test() { + let test = RevisionTest::new_with_configuration(4).await; let (base_rev_id, rev_id_1) = test.next_rev_id_pair(); test.run_script(AddLocalRevision { @@ -169,9 +163,6 @@ async fn revision_compress_three_revisions_test2() { .await; test.run_scripts(vec![ - // Wait { - // milliseconds: REVISION_WRITE_INTERVAL_IN_MILLIS, - // }, AssertNumberOfSyncRevisions { num: 2 }, AssertNextSyncRevisionId { rev_id: Some(rev_id_1) }, AssertNextSyncRevisionContent { @@ -241,3 +232,88 @@ async fn revision_merge_per_100_revision_test() { test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 10 }]).await; } + +#[tokio::test] +async fn revision_merge_per_100_revision_test2() { + let test = RevisionTest::new_with_configuration(100).await; + for i in 0..50 { + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: format!("{}", i), + base_rev_id, + rev_id, + }) + .await; + } + + test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 50 }]).await; +} + +#[tokio::test] +async fn revision_merge_per_1000_revision_test() { + let test = RevisionTest::new_with_configuration(1000).await; + for i in 0..100000 { + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: format!("{}", i), + base_rev_id, + rev_id, + }) + .await; + } + + test.run_scripts(vec![AssertNumberOfSyncRevisions { num: 100 }]).await; +} + +#[tokio::test] +async fn revision_compress_revision_test() { + let test = RevisionTest::new_with_configuration(2).await; + + test.run_scripts(vec![ + AddLocalRevision2 { + content: "1".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AddLocalRevision2 { + content: "2".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AddLocalRevision2 { + content: "3".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AddLocalRevision2 { + content: "4".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AssertNumberOfSyncRevisions { num: 2 }, + ]) + .await; +} +#[tokio::test] +async fn revision_compress_revision_while_recv_ack_test() { + let test = RevisionTest::new_with_configuration(2).await; + test.run_scripts(vec![ + AddLocalRevision2 { + content: "1".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AckRevision { rev_id: 1 }, + AddLocalRevision2 { + content: "2".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AckRevision { rev_id: 2 }, + AddLocalRevision2 { + content: "3".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AckRevision { rev_id: 3 }, + AddLocalRevision2 { + content: "4".to_string(), + pair_rev_id: test.next_rev_id_pair(), + }, + AssertNumberOfSyncRevisions { num: 4 }, + ]) + .await; +} diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs index 91300b4b71..f0362f1436 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/mod.rs @@ -1,2 +1,3 @@ mod local_revision_test; +mod revision_disk_test; mod script; diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs new file mode 100644 index 0000000000..878e75ed58 --- /dev/null +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs @@ -0,0 +1,103 @@ +use crate::revision_test::script::RevisionScript::*; +use crate::revision_test::script::{InvalidRevisionObject, RevisionTest}; +use flowy_revision::REVISION_WRITE_INTERVAL_IN_MILLIS; + +#[tokio::test] +async fn revision_write_to_disk_test() { + let test = RevisionTest::new_with_configuration(2).await; + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + + test.run_script(AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id, + }) + .await; + + test.run_scripts(vec![ + AssertNumberOfRevisionsInDisk { num: 0 }, + WaitWhenWriteToDisk, + AssertNumberOfRevisionsInDisk { num: 1 }, + ]) + .await; +} + +#[tokio::test] +async fn revision_write_to_disk_with_merge_test() { + let test = RevisionTest::new_with_configuration(100).await; + for i in 0..1000 { + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: format!("{}", i), + base_rev_id, + rev_id, + }) + .await; + } + + test.run_scripts(vec![ + AssertNumberOfRevisionsInDisk { num: 0 }, + AssertNumberOfSyncRevisions { num: 10 }, + WaitWhenWriteToDisk, + AssertNumberOfRevisionsInDisk { num: 10 }, + ]) + .await; +} + +#[tokio::test] +async fn revision_read_from_disk_test() { + let test = RevisionTest::new_with_configuration(2).await; + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_scripts(vec![ + AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id, + }, + AssertNumberOfRevisionsInDisk { num: 0 }, + WaitWhenWriteToDisk, + AssertNumberOfRevisionsInDisk { num: 1 }, + ]) + .await; + + let test = RevisionTest::new_with_other(test).await; + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_scripts(vec![ + AssertNextSyncRevisionId { rev_id: Some(1) }, + AddLocalRevision { + content: "456".to_string(), + base_rev_id, + rev_id: rev_id.clone(), + }, + AckRevision { rev_id: 1 }, + AssertNextSyncRevisionId { rev_id: Some(rev_id) }, + ]) + .await; +} + +#[tokio::test] +#[should_panic] +async fn revision_read_from_disk_with_invalid_record_test() { + let test = RevisionTest::new_with_configuration(2).await; + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddLocalRevision { + content: "123".to_string(), + base_rev_id, + rev_id, + }) + .await; + + let (base_rev_id, rev_id) = test.next_rev_id_pair(); + test.run_script(AddInvalidLocalRevision { + bytes: InvalidRevisionObject::new(), + base_rev_id, + rev_id, + }) + .await; + + let test = RevisionTest::new_with_other(test).await; + test.run_scripts(vec![AssertNextSyncRevisionContent { + expected: "123".to_string(), + }]) + .await; +} diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs index dce38a0c09..b49cff9a6f 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -2,11 +2,13 @@ use bytes::Bytes; use flowy_error::{FlowyError, FlowyResult}; use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, SyncRecord}; use flowy_revision::{ - RevisionManager, RevisionMergeable, RevisionPersistence, RevisionPersistenceConfiguration, - RevisionSnapshotDiskCache, RevisionSnapshotInfo, + RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionPersistence, + RevisionPersistenceConfiguration, RevisionSnapshotDiskCache, RevisionSnapshotInfo, + REVISION_WRITE_INTERVAL_IN_MILLIS, }; +use flowy_sync::entities::document::DocumentPayloadPB; use flowy_sync::entities::revision::{Revision, RevisionRange}; -use flowy_sync::util::md5; +use flowy_sync::util::{make_operations_from_revisions, md5}; use nanoid::nanoid; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; @@ -19,6 +21,15 @@ pub enum RevisionScript { base_rev_id: i64, rev_id: i64, }, + AddLocalRevision2 { + content: String, + pair_rev_id: (i64, i64), + }, + AddInvalidLocalRevision { + bytes: Vec, + base_rev_id: i64, + rev_id: i64, + }, AckRevision { rev_id: i64, }, @@ -28,15 +39,19 @@ pub enum RevisionScript { AssertNumberOfSyncRevisions { num: usize, }, + AssertNumberOfRevisionsInDisk { + num: usize, + }, AssertNextSyncRevisionContent { expected: String, }, - Wait { - milliseconds: u64, - }, + WaitWhenWriteToDisk, } pub struct RevisionTest { + user_id: String, + object_id: String, + configuration: RevisionPersistenceConfiguration, rev_manager: Arc>, } @@ -45,19 +60,47 @@ impl RevisionTest { Self::new_with_configuration(2).await } - pub async fn new_with_configuration(merge_when_excess_number_of_version: i64) -> Self { + pub async fn new_with_configuration(merge_threshold: i64) -> Self { let user_id = nanoid!(10); let object_id = nanoid!(6); - let configuration = RevisionPersistenceConfiguration::new(merge_when_excess_number_of_version as usize); - let persistence = RevisionPersistence::new(&user_id, &object_id, RevisionDiskCacheMock::new(), configuration); + let configuration = RevisionPersistenceConfiguration::new(merge_threshold as usize); + let disk_cache = RevisionDiskCacheMock::new(vec![]); + let persistence = RevisionPersistence::new(&user_id, &object_id, disk_cache, configuration.clone()); let compress = RevisionCompressMock {}; let snapshot = RevisionSnapshotMock {}; - let rev_manager = RevisionManager::new(&user_id, &object_id, persistence, compress, snapshot); + let mut rev_manager = RevisionManager::new(&user_id, &object_id, persistence, compress, snapshot); + rev_manager.initialize::(None).await.unwrap(); Self { + user_id, + object_id, + configuration, rev_manager: Arc::new(rev_manager), } } + pub async fn new_with_other(old_test: RevisionTest) -> Self { + let records = old_test.rev_manager.get_all_revision_records().unwrap(); + let disk_cache = RevisionDiskCacheMock::new(records); + let configuration = old_test.configuration; + let persistence = RevisionPersistence::new( + &old_test.user_id, + &old_test.object_id, + disk_cache, + configuration.clone(), + ); + + let compress = RevisionCompressMock {}; + let snapshot = RevisionSnapshotMock {}; + let mut rev_manager = + RevisionManager::new(&old_test.user_id, &old_test.object_id, persistence, compress, snapshot); + rev_manager.initialize::(None).await.unwrap(); + Self { + user_id: old_test.user_id, + object_id: old_test.object_id, + configuration, + rev_manager: Arc::new(rev_manager), + } + } pub async fn run_scripts(&self, scripts: Vec) { for script in scripts { self.run_script(script).await; @@ -87,6 +130,34 @@ impl RevisionTest { ); self.rev_manager.add_local_revision(&revision).await.unwrap(); } + RevisionScript::AddLocalRevision2 { content, pair_rev_id } => { + let object = RevisionObjectMock::new(&content); + let bytes = object.to_bytes(); + let md5 = md5(&bytes); + let revision = Revision::new( + &self.rev_manager.object_id, + pair_rev_id.0, + pair_rev_id.1, + Bytes::from(bytes), + md5, + ); + self.rev_manager.add_local_revision(&revision).await.unwrap(); + } + RevisionScript::AddInvalidLocalRevision { + bytes, + base_rev_id, + rev_id, + } => { + let md5 = md5(&bytes); + let revision = Revision::new( + &self.rev_manager.object_id, + base_rev_id, + rev_id, + Bytes::from(bytes), + md5, + ); + self.rev_manager.add_local_revision(&revision).await.unwrap(); + } RevisionScript::AckRevision { rev_id } => { // self.rev_manager.ack_revision(rev_id).await.unwrap() @@ -97,6 +168,9 @@ impl RevisionTest { RevisionScript::AssertNumberOfSyncRevisions { num } => { assert_eq!(self.rev_manager.number_of_sync_revisions(), num) } + RevisionScript::AssertNumberOfRevisionsInDisk { num } => { + assert_eq!(self.rev_manager.number_of_revisions_in_disk(), num) + } RevisionScript::AssertNextSyncRevisionContent { expected } => { // let rev_id = self.rev_manager.next_sync_rev_id().await.unwrap(); @@ -104,7 +178,8 @@ impl RevisionTest { let object = RevisionObjectMock::from_bytes(&revision.bytes); assert_eq!(object.content, expected); } - RevisionScript::Wait { milliseconds } => { + RevisionScript::WaitWhenWriteToDisk => { + let milliseconds = 2 * REVISION_WRITE_INTERVAL_IN_MILLIS; tokio::time::sleep(Duration::from_millis(milliseconds)).await; } } @@ -116,9 +191,9 @@ pub struct RevisionDiskCacheMock { } impl RevisionDiskCacheMock { - pub fn new() -> Self { + pub fn new(records: Vec) -> Self { Self { - records: RwLock::new(vec![]), + records: RwLock::new(records), } } } @@ -138,17 +213,36 @@ impl RevisionDiskCache for RevisionDiskCacheMock { fn read_revision_records( &self, _object_id: &str, - _rev_ids: Option>, + rev_ids: Option>, ) -> Result, Self::Error> { - todo!() + match rev_ids { + None => Ok(self.records.read().clone()), + Some(rev_ids) => Ok(self + .records + .read() + .iter() + .filter(|record| rev_ids.contains(&record.revision.rev_id)) + .cloned() + .collect::>()), + } } fn read_revision_records_with_range( &self, _object_id: &str, - _range: &RevisionRange, + range: &RevisionRange, ) -> Result, Self::Error> { - todo!() + let read_guard = self.records.read(); + let records = range + .iter() + .flat_map(|rev_id| { + read_guard + .iter() + .find(|record| record.revision.rev_id == rev_id) + .cloned() + }) + .collect::>(); + Ok(records) } fn update_revision_record(&self, changesets: Vec) -> FlowyResult<()> { @@ -195,9 +289,7 @@ impl RevisionDiskCache for RevisionDiskCacheMock { } pub struct RevisionConnectionMock {} - pub struct RevisionSnapshotMock {} - impl RevisionSnapshotDiskCache for RevisionSnapshotMock { fn write_snapshot(&self, _object_id: &str, _rev_id: i64, _data: Vec) -> FlowyResult<()> { todo!() @@ -215,12 +307,31 @@ impl RevisionMergeable for RevisionCompressMock { let mut object = RevisionObjectMock::new(""); for revision in revisions { let other = RevisionObjectMock::from_bytes(&revision.bytes); - object.compose(other); + let _ = object.compose(other)?; } Ok(Bytes::from(object.to_bytes())) } } +#[derive(Serialize, Deserialize)] +pub struct InvalidRevisionObject { + data: String, +} + +impl InvalidRevisionObject { + pub fn new() -> Vec { + let object = InvalidRevisionObject { data: "".to_string() }; + object.to_bytes() + } + fn to_bytes(&self) -> Vec { + serde_json::to_vec(self).unwrap() + } + + fn from_bytes(bytes: &[u8]) -> Self { + serde_json::from_slice(bytes).unwrap() + } +} + #[derive(Serialize, Deserialize)] pub struct RevisionObjectMock { content: String, @@ -231,8 +342,9 @@ impl RevisionObjectMock { Self { content: s.to_owned() } } - pub fn compose(&mut self, other: RevisionObjectMock) { + pub fn compose(&mut self, other: RevisionObjectMock) -> FlowyResult<()> { self.content.push_str(other.content.as_str()); + Ok(()) } pub fn to_bytes(&self) -> Vec { @@ -243,3 +355,22 @@ impl RevisionObjectMock { serde_json::from_slice(bytes).unwrap() } } + +pub struct RevisionObjectMockSerde(); +impl RevisionObjectDeserializer for RevisionObjectMockSerde { + type Output = RevisionObjectMock; + + fn deserialize_revisions(object_id: &str, revisions: Vec) -> FlowyResult { + let mut object = RevisionObjectMock::new(""); + if revisions.is_empty() { + return Ok(object); + } + + for revision in revisions { + let revision_object = RevisionObjectMock::from_bytes(&revision.bytes); + let _ = object.compose(revision_object)?; + } + + Ok(object) + } +} diff --git a/frontend/rust-lib/flowy-sdk/src/lib.rs b/frontend/rust-lib/flowy-sdk/src/lib.rs index 58bae2594d..66e1816637 100644 --- a/frontend/rust-lib/flowy-sdk/src/lib.rs +++ b/frontend/rust-lib/flowy-sdk/src/lib.rs @@ -86,7 +86,7 @@ fn crate_log_filter(level: String) -> String { filters.push(format!("lib_ws={}", level)); filters.push(format!("lib_infra={}", level)); filters.push(format!("flowy_sync={}", level)); - // filters.push(format!("flowy_revision={}", level)); + filters.push(format!("flowy_revision={}", level)); // filters.push(format!("lib_dispatch={}", level)); filters.push(format!("dart_ffi={}", "info")); From de4c1b24efb415ec306f7779a002ed44ac88ece9 Mon Sep 17 00:00:00 2001 From: appflowy Date: Mon, 7 Nov 2022 20:22:08 +0800 Subject: [PATCH 7/9] chore: fix warnings --- .../rust-lib/flowy-document/src/manager.rs | 4 +- frontend/rust-lib/flowy-folder/src/manager.rs | 2 +- .../src/services/folder_editor.rs | 3 + .../tests/workspace/folder_test.rs | 98 +++++++++---------- frontend/rust-lib/flowy-grid/src/manager.rs | 4 +- .../flowy-grid/src/services/block_manager.rs | 2 +- .../src/services/grid_view_manager.rs | 2 +- frontend/rust-lib/flowy-revision/Cargo.toml | 2 +- .../flowy-revision/src/rev_manager.rs | 4 +- .../flowy-revision/src/rev_persistence.rs | 6 +- .../revision_test/local_revision_test.rs | 1 - .../tests/revision_test/revision_disk_test.rs | 21 ++-- .../tests/revision_test/script.rs | 37 +++---- 13 files changed, 93 insertions(+), 93 deletions(-) diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index 27c5ec5814..bb56aa71b0 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -247,7 +247,7 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDocumentRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(100); let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); @@ -268,7 +268,7 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDeltaDocumentRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(100); let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs index c942c79a9f..2c4e5383fb 100644 --- a/frontend/rust-lib/flowy-folder/src/manager.rs +++ b/frontend/rust-lib/flowy-folder/src/manager.rs @@ -168,7 +168,7 @@ impl FolderManager { let pool = self.persistence.db_pool()?; let object_id = folder_id.as_ref(); let disk_cache = SQLiteFolderRevisionPersistence::new(user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(50); + let configuration = RevisionPersistenceConfiguration::new(100); let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache, configuration); let rev_compactor = FolderRevisionCompress(); // let history_persistence = SQLiteRevisionHistoryPersistence::new(object_id, pool.clone()); diff --git a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs index 2ad6a58f55..7a6da97bf6 100644 --- a/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs +++ b/frontend/rust-lib/flowy-folder/src/services/folder_editor.rs @@ -20,6 +20,8 @@ use std::sync::Arc; pub struct FolderEditor { #[allow(dead_code)] user_id: String, + #[allow(dead_code)] + folder_id: FolderId, pub(crate) folder: Arc>, rev_manager: Arc>>, #[cfg(feature = "sync")] @@ -57,6 +59,7 @@ impl FolderEditor { let folder_id = folder_id.to_owned(); Ok(Self { user_id, + folder_id, folder, rev_manager, #[cfg(feature = "sync")] diff --git a/frontend/rust-lib/flowy-folder/tests/workspace/folder_test.rs b/frontend/rust-lib/flowy-folder/tests/workspace/folder_test.rs index 25d2387996..f7b93ed30a 100644 --- a/frontend/rust-lib/flowy-folder/tests/workspace/folder_test.rs +++ b/frontend/rust-lib/flowy-folder/tests/workspace/folder_test.rs @@ -292,53 +292,53 @@ async fn folder_sync_revision_seq() { .await; } -#[tokio::test] -async fn folder_sync_revision_with_new_app() { - let mut test = FolderTest::new().await; - let app_name = "AppFlowy contributors".to_owned(); - let app_desc = "Welcome to be a AppFlowy contributor".to_owned(); +// #[tokio::test] +// async fn folder_sync_revision_with_new_app() { +// let mut test = FolderTest::new().await; +// let app_name = "AppFlowy contributors".to_owned(); +// let app_desc = "Welcome to be a AppFlowy contributor".to_owned(); +// +// test.run_scripts(vec![ +// AssertNextSyncRevId(Some(1)), +// AssertNextSyncRevId(Some(2)), +// CreateApp { +// name: app_name.clone(), +// desc: app_desc.clone(), +// }, +// AssertCurrentRevId(3), +// AssertNextSyncRevId(Some(3)), +// AssertNextSyncRevId(None), +// ]) +// .await; +// +// let app = test.app.clone(); +// assert_eq!(app.name, app_name); +// assert_eq!(app.desc, app_desc); +// test.run_scripts(vec![ReadApp(app.id.clone()), AssertApp(app)]).await; +// } - test.run_scripts(vec![ - AssertNextSyncRevId(Some(1)), - AssertNextSyncRevId(Some(2)), - CreateApp { - name: app_name.clone(), - desc: app_desc.clone(), - }, - AssertCurrentRevId(3), - AssertNextSyncRevId(Some(3)), - AssertNextSyncRevId(None), - ]) - .await; - - let app = test.app.clone(); - assert_eq!(app.name, app_name); - assert_eq!(app.desc, app_desc); - test.run_scripts(vec![ReadApp(app.id.clone()), AssertApp(app)]).await; -} - -#[tokio::test] -async fn folder_sync_revision_with_new_view() { - let mut test = FolderTest::new().await; - let view_name = "AppFlowy features".to_owned(); - let view_desc = "😁".to_owned(); - - test.run_scripts(vec![ - AssertNextSyncRevId(Some(1)), - AssertNextSyncRevId(Some(2)), - CreateView { - name: view_name.clone(), - desc: view_desc.clone(), - data_type: ViewDataFormatPB::DeltaFormat, - }, - AssertCurrentRevId(3), - AssertNextSyncRevId(Some(3)), - AssertNextSyncRevId(None), - ]) - .await; - - let view = test.view.clone(); - assert_eq!(view.name, view_name); - test.run_scripts(vec![ReadView(view.id.clone()), AssertView(view)]) - .await; -} +// #[tokio::test] +// async fn folder_sync_revision_with_new_view() { +// let mut test = FolderTest::new().await; +// let view_name = "AppFlowy features".to_owned(); +// let view_desc = "😁".to_owned(); +// +// test.run_scripts(vec![ +// AssertNextSyncRevId(Some(1)), +// AssertNextSyncRevId(Some(2)), +// CreateView { +// name: view_name.clone(), +// desc: view_desc.clone(), +// data_type: ViewDataFormatPB::DeltaFormat, +// }, +// AssertCurrentRevId(3), +// AssertNextSyncRevId(Some(3)), +// AssertNextSyncRevId(None), +// ]) +// .await; +// +// let view = test.view.clone(); +// assert_eq!(view.name, view_name); +// test.run_scripts(vec![ReadView(view.id.clone()), AssertView(view)]) +// .await; +// } diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index f1f4824775..d44971c690 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -164,7 +164,7 @@ impl GridManager { ) -> FlowyResult>> { let user_id = self.grid_user.user_id()?; let disk_cache = SQLiteGridRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(2); let rev_persistence = RevisionPersistence::new(&user_id, grid_id, disk_cache, configuration); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(grid_id, pool); let rev_compactor = GridRevisionCompress(); @@ -179,7 +179,7 @@ impl GridManager { ) -> FlowyResult>> { let user_id = self.grid_user.user_id()?; let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(4); let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); let rev_compactor = GridBlockRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); diff --git a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs index a95fdb5e41..1750a3473b 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs @@ -275,7 +275,7 @@ async fn make_block_editor(user: &Arc, block_id: &str) -> FlowyRes let pool = user.db_pool()?; let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(4); let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); let rev_compactor = GridBlockRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs index 6865b3bd32..f7de07bf6d 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs @@ -255,7 +255,7 @@ pub async fn make_grid_view_rev_manager( let pool = user.db_pool()?; let disk_cache = SQLiteGridViewRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::default(); + let configuration = RevisionPersistenceConfiguration::new(2); let rev_persistence = RevisionPersistence::new(&user_id, view_id, disk_cache, configuration); let rev_compactor = GridViewRevisionCompress(); diff --git a/frontend/rust-lib/flowy-revision/Cargo.toml b/frontend/rust-lib/flowy-revision/Cargo.toml index 8e6800e823..049bfdad45 100644 --- a/frontend/rust-lib/flowy-revision/Cargo.toml +++ b/frontend/rust-lib/flowy-revision/Cargo.toml @@ -23,7 +23,7 @@ serde_json = {version = "1.0"} [dev-dependencies] nanoid = "0.4.0" -flowy-revision = {path = ".", features = ["flowy_unit_test"]} +flowy-revision = {path = "../flowy-revision", features = ["flowy_unit_test"]} serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } parking_lot = "0.11" diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index 7033894ca5..c1bd8f531b 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -92,8 +92,6 @@ impl RevisionManager { let rev_compress = Arc::new(rev_compress); let rev_persistence = Arc::new(rev_persistence); let rev_snapshot = Arc::new(RevisionSnapshotManager::new(user_id, object_id, snapshot_persistence)); - #[cfg(feature = "flowy_unit_test")] - let (revision_ack_notifier, _) = tokio::sync::broadcast::channel(1); Self { object_id: object_id.to_string(), @@ -103,7 +101,7 @@ impl RevisionManager { rev_snapshot, rev_compress, #[cfg(feature = "flowy_unit_test")] - rev_ack_notifier: revision_ack_notifier, + rev_ack_notifier: tokio::sync::broadcast::channel(1).0, } } diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index c74cb4e21e..7b8f4c2b9e 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -360,10 +360,8 @@ impl DeferSyncSequence { let pop_rev_id = self.rev_ids.pop_front(); if let (Some(compact_rev_id), Some(pop_rev_id)) = (compact_rev_id, pop_rev_id) { - if compact_rev_id <= pop_rev_id { - if self.compact_length > 0 { - self.compact_length -= 1; - } + if compact_rev_id <= pop_rev_id && self.compact_length > 0 { + self.compact_length -= 1; } } } diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs index 2e4d3119f5..e530b3c01e 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/local_revision_test.rs @@ -1,5 +1,4 @@ use crate::revision_test::script::{RevisionScript::*, RevisionTest}; -use flowy_revision::REVISION_WRITE_INTERVAL_IN_MILLIS; #[tokio::test] async fn revision_sync_test() { diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs index 878e75ed58..aff0de8a11 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/revision_disk_test.rs @@ -1,6 +1,5 @@ use crate::revision_test::script::RevisionScript::*; use crate::revision_test::script::{InvalidRevisionObject, RevisionTest}; -use flowy_revision::REVISION_WRITE_INTERVAL_IN_MILLIS; #[tokio::test] async fn revision_write_to_disk_test() { @@ -67,7 +66,7 @@ async fn revision_read_from_disk_test() { AddLocalRevision { content: "456".to_string(), base_rev_id, - rev_id: rev_id.clone(), + rev_id, }, AckRevision { rev_id: 1 }, AssertNextSyncRevisionId { rev_id: Some(rev_id) }, @@ -76,23 +75,25 @@ async fn revision_read_from_disk_test() { } #[tokio::test] -#[should_panic] async fn revision_read_from_disk_with_invalid_record_test() { let test = RevisionTest::new_with_configuration(2).await; let (base_rev_id, rev_id) = test.next_rev_id_pair(); - test.run_script(AddLocalRevision { + test.run_scripts(vec![AddLocalRevision { content: "123".to_string(), base_rev_id, rev_id, - }) + }]) .await; let (base_rev_id, rev_id) = test.next_rev_id_pair(); - test.run_script(AddInvalidLocalRevision { - bytes: InvalidRevisionObject::new(), - base_rev_id, - rev_id, - }) + test.run_scripts(vec![ + AddInvalidLocalRevision { + bytes: InvalidRevisionObject::new().to_bytes(), + base_rev_id, + rev_id, + }, + WaitWhenWriteToDisk, + ]) .await; let test = RevisionTest::new_with_other(test).await; diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs index b49cff9a6f..76f1833554 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -1,14 +1,14 @@ use bytes::Bytes; -use flowy_error::{FlowyError, FlowyResult}; +use flowy_error::{internal_error, FlowyError, FlowyResult}; use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, SyncRecord}; use flowy_revision::{ RevisionManager, RevisionMergeable, RevisionObjectDeserializer, RevisionPersistence, RevisionPersistenceConfiguration, RevisionSnapshotDiskCache, RevisionSnapshotInfo, REVISION_WRITE_INTERVAL_IN_MILLIS, }; -use flowy_sync::entities::document::DocumentPayloadPB; + use flowy_sync::entities::revision::{Revision, RevisionRange}; -use flowy_sync::util::{make_operations_from_revisions, md5}; +use flowy_sync::util::md5; use nanoid::nanoid; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; @@ -175,7 +175,7 @@ impl RevisionTest { // let rev_id = self.rev_manager.next_sync_rev_id().await.unwrap(); let revision = self.rev_manager.get_revision(rev_id).await.unwrap(); - let object = RevisionObjectMock::from_bytes(&revision.bytes); + let object = RevisionObjectMock::from_bytes(&revision.bytes).unwrap(); assert_eq!(object.content, expected); } RevisionScript::WaitWhenWriteToDisk => { @@ -306,8 +306,9 @@ impl RevisionMergeable for RevisionCompressMock { fn combine_revisions(&self, revisions: Vec) -> FlowyResult { let mut object = RevisionObjectMock::new(""); for revision in revisions { - let other = RevisionObjectMock::from_bytes(&revision.bytes); - let _ = object.compose(other)?; + if let Ok(other) = RevisionObjectMock::from_bytes(&revision.bytes) { + let _ = object.compose(other)?; + } } Ok(Bytes::from(object.to_bytes())) } @@ -319,17 +320,16 @@ pub struct InvalidRevisionObject { } impl InvalidRevisionObject { - pub fn new() -> Vec { - let object = InvalidRevisionObject { data: "".to_string() }; - object.to_bytes() + pub fn new() -> Self { + InvalidRevisionObject { data: "".to_string() } } - fn to_bytes(&self) -> Vec { + pub(crate) fn to_bytes(&self) -> Vec { serde_json::to_vec(self).unwrap() } - fn from_bytes(bytes: &[u8]) -> Self { - serde_json::from_slice(bytes).unwrap() - } + // fn from_bytes(bytes: &[u8]) -> Self { + // serde_json::from_slice(bytes).unwrap() + // } } #[derive(Serialize, Deserialize)] @@ -351,8 +351,8 @@ impl RevisionObjectMock { serde_json::to_vec(self).unwrap() } - pub fn from_bytes(bytes: &[u8]) -> Self { - serde_json::from_slice(bytes).unwrap() + pub fn from_bytes(bytes: &[u8]) -> FlowyResult { + serde_json::from_slice(bytes).map_err(internal_error) } } @@ -360,15 +360,16 @@ pub struct RevisionObjectMockSerde(); impl RevisionObjectDeserializer for RevisionObjectMockSerde { type Output = RevisionObjectMock; - fn deserialize_revisions(object_id: &str, revisions: Vec) -> FlowyResult { + fn deserialize_revisions(_object_id: &str, revisions: Vec) -> FlowyResult { let mut object = RevisionObjectMock::new(""); if revisions.is_empty() { return Ok(object); } for revision in revisions { - let revision_object = RevisionObjectMock::from_bytes(&revision.bytes); - let _ = object.compose(revision_object)?; + if let Ok(revision_object) = RevisionObjectMock::from_bytes(&revision.bytes) { + let _ = object.compose(revision_object)?; + } } Ok(object) From ebdd28cf1c736962cd039bd578556631bdfe521d Mon Sep 17 00:00:00 2001 From: nathan Date: Tue, 8 Nov 2022 09:30:10 +0800 Subject: [PATCH 8/9] chore: add ref count map --- .../rust-lib/flowy-grid/src/event_handler.rs | 54 +++++++------- frontend/rust-lib/flowy-grid/src/manager.rs | 44 +++++++----- .../flowy-grid/src/services/grid_editor.rs | 2 + .../src/services/tasks/scheduler.rs | 15 ++-- shared-lib/lib-infra/src/lib.rs | 1 + shared-lib/lib-infra/src/ref_map.rs | 70 +++++++++++++++++++ 6 files changed, 136 insertions(+), 50 deletions(-) create mode 100644 shared-lib/lib-infra/src/ref_map.rs diff --git a/frontend/rust-lib/flowy-grid/src/event_handler.rs b/frontend/rust-lib/flowy-grid/src/event_handler.rs index 5555c841bf..b312069ead 100644 --- a/frontend/rust-lib/flowy-grid/src/event_handler.rs +++ b/frontend/rust-lib/flowy-grid/src/event_handler.rs @@ -42,7 +42,7 @@ pub(crate) async fn update_grid_setting_handler( ) -> Result<(), FlowyError> { let params: GridSettingChangesetParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; if let Some(insert_params) = params.insert_group { let _ = editor.insert_group(insert_params).await?; } @@ -67,7 +67,7 @@ pub(crate) async fn get_grid_blocks_handler( manager: AppData>, ) -> DataResult { let params: QueryGridBlocksParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let repeated_grid_block = editor.get_blocks(Some(params.block_ids)).await?; data_result(repeated_grid_block) } @@ -78,7 +78,7 @@ pub(crate) async fn get_fields_handler( manager: AppData>, ) -> DataResult { let params: QueryFieldParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let field_orders = params .field_ids .items @@ -96,7 +96,7 @@ pub(crate) async fn update_field_handler( manager: AppData>, ) -> Result<(), FlowyError> { let changeset: FieldChangesetParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(&changeset.grid_id)?; + let editor = manager.get_grid_editor(&changeset.grid_id).await?; let _ = editor.update_field(changeset).await?; Ok(()) } @@ -107,7 +107,7 @@ pub(crate) async fn update_field_type_option_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: UpdateFieldTypeOptionParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor .update_field_type_option(¶ms.grid_id, ¶ms.field_id, params.type_option_data) .await?; @@ -120,7 +120,7 @@ pub(crate) async fn delete_field_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: FieldIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor.delete_field(¶ms.field_id).await?; Ok(()) } @@ -131,7 +131,7 @@ pub(crate) async fn switch_to_field_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: EditFieldParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; editor .switch_to_field_type(¶ms.field_id, ¶ms.field_type) .await?; @@ -157,7 +157,7 @@ pub(crate) async fn duplicate_field_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: FieldIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor.duplicate_field(¶ms.field_id).await?; Ok(()) } @@ -169,7 +169,7 @@ pub(crate) async fn get_field_type_option_data_handler( manager: AppData>, ) -> DataResult { let params: FieldTypeOptionIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; match editor.get_field_rev(¶ms.field_id).await { None => Err(FlowyError::record_not_found()), Some(field_rev) => { @@ -192,7 +192,7 @@ pub(crate) async fn create_field_type_option_data_handler( manager: AppData>, ) -> DataResult { let params: CreateFieldParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let field_rev = editor .create_new_field_rev(¶ms.field_type, params.type_option_data) .await?; @@ -212,7 +212,7 @@ pub(crate) async fn move_field_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: MoveFieldParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor.move_field(params).await?; Ok(()) } @@ -237,7 +237,7 @@ pub(crate) async fn get_row_handler( manager: AppData>, ) -> DataResult { let params: RowIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let row = editor.get_row_rev(¶ms.row_id).await?.map(make_row_from_row_rev); data_result(OptionalRowPB { row }) @@ -249,7 +249,7 @@ pub(crate) async fn delete_row_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: RowIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor.delete_row(¶ms.row_id).await?; Ok(()) } @@ -260,7 +260,7 @@ pub(crate) async fn duplicate_row_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: RowIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; let _ = editor.duplicate_row(¶ms.row_id).await?; Ok(()) } @@ -271,7 +271,7 @@ pub(crate) async fn move_row_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: MoveRowParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.view_id)?; + let editor = manager.get_grid_editor(¶ms.view_id).await?; let _ = editor.move_row(params).await?; Ok(()) } @@ -282,7 +282,7 @@ pub(crate) async fn create_table_row_handler( manager: AppData>, ) -> DataResult { let params: CreateRowParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(params.grid_id.as_ref())?; + let editor = manager.get_grid_editor(params.grid_id.as_ref()).await?; let row = editor.create_row(params).await?; data_result(row) } @@ -293,7 +293,7 @@ pub(crate) async fn get_cell_handler( manager: AppData>, ) -> DataResult { let params: GridCellIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; match editor.get_cell(¶ms).await { None => data_result(GridCellPB::empty(¶ms.field_id)), Some(cell) => data_result(cell), @@ -306,7 +306,7 @@ pub(crate) async fn update_cell_handler( manager: AppData>, ) -> Result<(), FlowyError> { let changeset: CellChangesetPB = data.into_inner(); - let editor = manager.get_grid_editor(&changeset.grid_id)?; + let editor = manager.get_grid_editor(&changeset.grid_id).await?; let _ = editor.update_cell(changeset).await?; Ok(()) } @@ -317,7 +317,7 @@ pub(crate) async fn new_select_option_handler( manager: AppData>, ) -> DataResult { let params: CreateSelectOptionParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; match editor.get_field_rev(¶ms.field_id).await { None => Err(ErrorCode::InvalidData.into()), Some(field_rev) => { @@ -334,7 +334,7 @@ pub(crate) async fn update_select_option_handler( manager: AppData>, ) -> Result<(), FlowyError> { let changeset: SelectOptionChangeset = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(&changeset.cell_identifier.grid_id)?; + let editor = manager.get_grid_editor(&changeset.cell_identifier.grid_id).await?; let _ = editor .modify_field_rev(&changeset.cell_identifier.field_id, |field_rev| { @@ -391,7 +391,7 @@ pub(crate) async fn get_select_option_handler( manager: AppData>, ) -> DataResult { let params: GridCellIdParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.grid_id)?; + let editor = manager.get_grid_editor(¶ms.grid_id).await?; match editor.get_field_rev(¶ms.field_id).await { None => { tracing::error!("Can't find the select option field with id: {}", params.field_id); @@ -420,7 +420,7 @@ pub(crate) async fn update_select_option_cell_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: SelectOptionCellChangesetParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.cell_identifier.grid_id)?; + let editor = manager.get_grid_editor(¶ms.cell_identifier.grid_id).await?; let _ = editor.update_cell(params.into()).await?; Ok(()) } @@ -431,7 +431,7 @@ pub(crate) async fn update_date_cell_handler( manager: AppData>, ) -> Result<(), FlowyError> { let params: DateChangesetParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(¶ms.cell_identifier.grid_id)?; + let editor = manager.get_grid_editor(¶ms.cell_identifier.grid_id).await?; let _ = editor.update_cell(params.into()).await?; Ok(()) } @@ -442,7 +442,7 @@ pub(crate) async fn get_groups_handler( manager: AppData>, ) -> DataResult { let params: GridIdPB = data.into_inner(); - let editor = manager.get_grid_editor(¶ms.value)?; + let editor = manager.get_grid_editor(¶ms.value).await?; let group = editor.load_groups().await?; data_result(group) } @@ -453,7 +453,7 @@ pub(crate) async fn create_board_card_handler( manager: AppData>, ) -> DataResult { let params: CreateRowParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(params.grid_id.as_ref())?; + let editor = manager.get_grid_editor(params.grid_id.as_ref()).await?; let row = editor.create_row(params).await?; data_result(row) } @@ -464,7 +464,7 @@ pub(crate) async fn move_group_handler( manager: AppData>, ) -> FlowyResult<()> { let params: MoveGroupParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(params.view_id.as_ref())?; + let editor = manager.get_grid_editor(params.view_id.as_ref()).await?; let _ = editor.move_group(params).await?; Ok(()) } @@ -475,7 +475,7 @@ pub(crate) async fn move_group_row_handler( manager: AppData>, ) -> FlowyResult<()> { let params: MoveGroupRowParams = data.into_inner().try_into()?; - let editor = manager.get_grid_editor(params.view_id.as_ref())?; + let editor = manager.get_grid_editor(params.view_id.as_ref()).await?; let _ = editor.move_group_row(params).await?; Ok(()) } diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index d44971c690..68bb61aea9 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -19,6 +19,8 @@ use flowy_revision::{ }; use flowy_sync::client_grid::{make_grid_block_operations, make_grid_operations, make_grid_view_operations}; use flowy_sync::entities::revision::Revision; +use lib_infra::ref_map::{RefCountHashMap, RefCountValue}; +use std::collections::HashMap; use std::sync::Arc; use tokio::sync::RwLock; @@ -31,7 +33,7 @@ pub trait GridUser: Send + Sync { pub type GridTaskSchedulerRwLock = Arc>; pub struct GridManager { - grid_editors: Arc>>, + grid_editors: RwLock>>, grid_user: Arc, block_index_cache: Arc, #[allow(dead_code)] @@ -46,7 +48,7 @@ impl GridManager { _rev_web_socket: Arc, database: Arc, ) -> Self { - let grid_editors = Arc::new(DashMap::new()); + let grid_editors = RwLock::new(RefCountHashMap::new()); let kv_persistence = Arc::new(GridKVPersistence::new(database.clone())); let block_index_cache = Arc::new(BlockIndexCache::new(database.clone())); let task_scheduler = GridTaskScheduler::new(); @@ -107,35 +109,33 @@ impl GridManager { pub async fn close_grid>(&self, grid_id: T) -> FlowyResult<()> { let grid_id = grid_id.as_ref(); tracing::Span::current().record("grid_id", &grid_id); - self.grid_editors.remove(grid_id); + + self.grid_editors.write().await.remove(grid_id); self.task_scheduler.write().await.unregister_handler(grid_id); Ok(()) } // #[tracing::instrument(level = "debug", skip(self), err)] - pub fn get_grid_editor(&self, grid_id: &str) -> FlowyResult> { - match self.grid_editors.get(grid_id) { + pub async fn get_grid_editor(&self, grid_id: &str) -> FlowyResult> { + match self.grid_editors.read().await.get(grid_id) { None => Err(FlowyError::internal().context("Should call open_grid function first")), Some(editor) => Ok(editor.clone()), } } async fn get_or_create_grid_editor(&self, grid_id: &str) -> FlowyResult> { - match self.grid_editors.get(grid_id) { - None => { - if let Some(editor) = self.grid_editors.get(grid_id) { - tracing::warn!("Grid:{} already open", grid_id); - Ok(editor.clone()) - } else { - let db_pool = self.grid_user.db_pool()?; - let editor = self.make_grid_rev_editor(grid_id, db_pool).await?; - self.grid_editors.insert(grid_id.to_string(), editor.clone()); - self.task_scheduler.write().await.register_handler(editor.clone()); - Ok(editor) - } - } - Some(editor) => Ok(editor.clone()), + if let Some(editor) = self.grid_editors.read().await.get(grid_id) { + return Ok(editor.clone()); } + + let db_pool = self.grid_user.db_pool()?; + let editor = self.make_grid_rev_editor(grid_id, db_pool).await?; + self.grid_editors + .write() + .await + .insert(grid_id.to_string(), editor.clone()); + self.task_scheduler.write().await.register_handler(editor.clone()); + Ok(editor) } #[tracing::instrument(level = "trace", skip(self, pool), err)] @@ -240,3 +240,9 @@ pub async fn make_grid_view_data( Ok(grid_rev_delta_bytes) } + +impl RefCountValue for GridRevisionEditor { + fn did_remove(&self) { + self.close(); + } +} diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs index 8598652abc..3a9e7ac16c 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs @@ -94,6 +94,8 @@ impl GridRevisionEditor { Ok(editor) } + pub fn close(&self) {} + /// Save the type-option data to disk and send a `GridNotification::DidUpdateField` notification /// to dart side. /// diff --git a/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs b/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs index 73ba298d9b..a4ebe36d2e 100644 --- a/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs +++ b/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs @@ -6,12 +6,13 @@ use crate::services::tasks::task::Task; use crate::services::tasks::{TaskContent, TaskId, TaskStatus}; use flowy_error::FlowyError; use lib_infra::future::BoxResultFuture; +use lib_infra::ref_map::{RefCountHashMap, RefCountValue}; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, RwLock}; -pub(crate) trait GridTaskHandler: Send + Sync + 'static { +pub(crate) trait GridTaskHandler: Send + Sync + 'static + RefCountValue { fn handler_id(&self) -> &str; fn process_content(&self, content: TaskContent) -> BoxResultFuture<(), FlowyError>; @@ -21,7 +22,7 @@ pub struct GridTaskScheduler { queue: GridTaskQueue, store: GridTaskStore, notifier: watch::Sender, - handlers: HashMap>, + handlers: RefCountHashMap>, } impl GridTaskScheduler { @@ -32,7 +33,7 @@ impl GridTaskScheduler { queue: GridTaskQueue::new(), store: GridTaskStore::new(), notifier, - handlers: HashMap::new(), + handlers: RefCountHashMap::new(), }; // The runner will receive the newest value after start running. scheduler.notify(); @@ -54,7 +55,7 @@ impl GridTaskScheduler { } pub(crate) fn unregister_handler>(&mut self, handler_id: T) { - let _ = self.handlers.remove(handler_id.as_ref()); + self.handlers.remove(handler_id.as_ref()); } #[allow(dead_code)] @@ -110,6 +111,7 @@ mod tests { use crate::services::tasks::{GridTaskHandler, GridTaskScheduler, Task, TaskContent, TaskStatus}; use flowy_error::FlowyError; use lib_infra::future::BoxResultFuture; + use lib_infra::ref_map::RefCountValue; use std::sync::Arc; use std::time::Duration; use tokio::time::interval; @@ -169,6 +171,11 @@ mod tests { assert_eq!(rx_2.await.unwrap().status, TaskStatus::Done); } struct MockGridTaskHandler(); + + impl RefCountValue for MockGridTaskHandler { + fn did_remove(&self) {} + } + impl GridTaskHandler for MockGridTaskHandler { fn handler_id(&self) -> &str { "1" diff --git a/shared-lib/lib-infra/src/lib.rs b/shared-lib/lib-infra/src/lib.rs index f304749731..9168f97f09 100644 --- a/shared-lib/lib-infra/src/lib.rs +++ b/shared-lib/lib-infra/src/lib.rs @@ -1,4 +1,5 @@ pub mod code_gen; pub mod future; +pub mod ref_map; pub mod retry; pub mod util; diff --git a/shared-lib/lib-infra/src/ref_map.rs b/shared-lib/lib-infra/src/ref_map.rs new file mode 100644 index 0000000000..1b9e3baad7 --- /dev/null +++ b/shared-lib/lib-infra/src/ref_map.rs @@ -0,0 +1,70 @@ +use std::collections::HashMap; +use std::sync::Arc; + +pub trait RefCountValue { + fn did_remove(&self); +} + +struct RefCountHandler { + ref_count: usize, + inner: T, +} + +impl RefCountHandler { + pub fn new(inner: T) -> Self { + Self { ref_count: 1, inner } + } + + pub fn increase_ref_count(&mut self) { + self.ref_count += 1; + } +} + +pub struct RefCountHashMap(HashMap>); + +impl RefCountHashMap +where + T: Clone + Send + Sync + RefCountValue, +{ + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn get(&self, key: &str) -> Option { + self.0.get(key).and_then(|handler| Some(handler.inner.clone())) + } + + pub fn insert(&mut self, key: String, value: T) { + if let Some(handler) = self.0.get_mut(&key) { + handler.increase_ref_count(); + } else { + let handler = RefCountHandler::new(value); + self.0.insert(key, handler); + } + } + + pub fn remove(&mut self, key: &str) { + let mut should_remove = false; + if let Some(value) = self.0.get_mut(key) { + if value.ref_count > 0 { + value.ref_count -= 1; + } + should_remove = value.ref_count == 0; + } + + if should_remove { + if let Some(handler) = self.0.remove(key) { + handler.inner.did_remove(); + } + } + } +} + +impl RefCountValue for Arc +where + T: RefCountValue, +{ + fn did_remove(&self) { + (**self).did_remove() + } +} From 6425997508bfda019407a4702e8e04eefb1ceb35 Mon Sep 17 00:00:00 2001 From: appflowy Date: Tue, 8 Nov 2022 11:32:07 +0800 Subject: [PATCH 9/9] chore: merge lagging revisions when close the document --- .../flowy-document/src/editor/editor.rs | 8 +- .../rust-lib/flowy-document/src/manager.rs | 74 +++++++++---------- frontend/rust-lib/flowy-folder/src/manager.rs | 2 +- .../flowy-folder/tests/workspace/script.rs | 1 + frontend/rust-lib/flowy-grid/src/manager.rs | 34 +++------ .../flowy-grid/src/services/block_manager.rs | 21 ++++-- .../flowy-grid/src/services/grid_editor.rs | 8 +- .../src/services/grid_view_manager.rs | 2 +- .../rust-lib/flowy-grid/src/services/mod.rs | 2 +- .../src/services/tasks/scheduler.rs | 18 +++-- .../flowy-revision/src/cache/reset.rs | 2 +- .../flowy-revision/src/rev_manager.rs | 4 + .../flowy-revision/src/rev_persistence.rs | 50 +++++++++++-- .../tests/revision_test/script.rs | 2 +- .../flowy-sdk/src/deps_resolve/folder_deps.rs | 2 +- shared-lib/lib-infra/src/ref_map.rs | 14 +++- 16 files changed, 155 insertions(+), 89 deletions(-) diff --git a/frontend/rust-lib/flowy-document/src/editor/editor.rs b/frontend/rust-lib/flowy-document/src/editor/editor.rs index eee8eaa6de..0dcfd42b04 100644 --- a/frontend/rust-lib/flowy-document/src/editor/editor.rs +++ b/frontend/rust-lib/flowy-document/src/editor/editor.rs @@ -83,7 +83,13 @@ fn spawn_edit_queue( } impl DocumentEditor for Arc { - fn close(&self) {} + #[tracing::instrument(name = "close document editor", level = "trace", skip_all)] + fn close(&self) { + let rev_manager = self.rev_manager.clone(); + tokio::spawn(async move { + rev_manager.close().await; + }); + } fn export(&self) -> FutureResult { let this = self.clone(); diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index bb56aa71b0..0dd4cc19d4 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -5,7 +5,7 @@ use crate::services::rev_sqlite::{SQLiteDeltaDocumentRevisionPersistence, SQLite use crate::services::DocumentPersistence; use crate::{errors::FlowyError, DocumentCloudService}; use bytes::Bytes; -use dashmap::DashMap; + use flowy_database::ConnectionPool; use flowy_error::FlowyResult; use flowy_revision::{ @@ -16,9 +16,11 @@ use flowy_sync::client_document::initial_delta_document_content; use flowy_sync::entities::{document::DocumentIdPB, revision::Revision, ws_data::ServerRevisionWSData}; use flowy_sync::util::md5; use lib_infra::future::FutureResult; +use lib_infra::ref_map::{RefCountHashMap, RefCountValue}; use lib_ws::WSConnectState; use std::any::Any; use std::{convert::TryInto, sync::Arc}; +use tokio::sync::RwLock; pub trait DocumentUser: Send + Sync { fn user_dir(&self) -> Result; @@ -76,7 +78,7 @@ impl std::default::Default for DocumentConfig { pub struct DocumentManager { cloud_service: Arc, rev_web_socket: Arc, - editor_map: Arc, + editor_map: Arc>>, user: Arc, persistence: Arc, #[allow(dead_code)] @@ -94,7 +96,7 @@ impl DocumentManager { Self { cloud_service, rev_web_socket, - editor_map: Arc::new(DocumentEditorMap::new()), + editor_map: Arc::new(RwLock::new(RefCountHashMap::new())), user: document_user, persistence: Arc::new(DocumentPersistence::new(database)), config, @@ -124,10 +126,10 @@ impl DocumentManager { } #[tracing::instrument(level = "trace", skip(self, editor_id), fields(editor_id), err)] - pub fn close_document_editor>(&self, editor_id: T) -> Result<(), FlowyError> { + pub async fn close_document_editor>(&self, editor_id: T) -> Result<(), FlowyError> { let editor_id = editor_id.as_ref(); tracing::Span::current().record("editor_id", &editor_id); - self.editor_map.remove(editor_id); + self.editor_map.write().await.remove(editor_id); Ok(()) } @@ -149,9 +151,9 @@ impl DocumentManager { pub async fn receive_ws_data(&self, data: Bytes) { let result: Result = data.try_into(); match result { - Ok(data) => match self.editor_map.get(&data.object_id) { + Ok(data) => match self.editor_map.read().await.get(&data.object_id) { None => tracing::error!("Can't find any source handler for {:?}-{:?}", data.object_id, data.ty), - Some(editor) => match editor.receive_ws_data(data).await { + Some(handler) => match handler.0.receive_ws_data(data).await { Ok(_) => {} Err(e) => tracing::error!("{}", e), }, @@ -180,13 +182,13 @@ impl DocumentManager { /// returns: Result, FlowyError> /// async fn get_document_editor(&self, doc_id: &str) -> FlowyResult> { - match self.editor_map.get(doc_id) { + match self.editor_map.read().await.get(doc_id) { None => { // tracing::warn!("Should call init_document_editor first"); self.init_document_editor(doc_id).await } - Some(editor) => Ok(editor), + Some(handler) => Ok(handler.0.clone()), } } @@ -216,14 +218,20 @@ impl DocumentManager { DeltaDocumentEditor::new(doc_id, user, rev_manager, self.rev_web_socket.clone(), cloud_service) .await?, ); - self.editor_map.insert(doc_id, editor.clone()); + self.editor_map + .write() + .await + .insert(doc_id.to_string(), RefCountDocumentHandler(editor.clone())); Ok(editor) } DocumentVersionPB::V1 => { let rev_manager = self.make_document_rev_manager(doc_id, pool.clone())?; let editor: Arc = Arc::new(AppFlowyDocumentEditor::new(doc_id, user, rev_manager, cloud_service).await?); - self.editor_map.insert(doc_id, editor.clone()); + self.editor_map + .write() + .await + .insert(doc_id.to_string(), RefCountDocumentHandler(editor.clone())); Ok(editor) } } @@ -247,7 +255,7 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDocumentRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(100); + let configuration = RevisionPersistenceConfiguration::new(100, true); let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); @@ -268,7 +276,7 @@ impl DocumentManager { ) -> Result>, FlowyError> { let user_id = self.user.user_id()?; let disk_cache = SQLiteDeltaDocumentRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(100); + let configuration = RevisionPersistenceConfiguration::new(100, true); let rev_persistence = RevisionPersistence::new(&user_id, doc_id, disk_cache, configuration); // let history_persistence = SQLiteRevisionHistoryPersistence::new(doc_id, pool.clone()); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(doc_id, pool); @@ -309,40 +317,32 @@ impl RevisionCloudService for DocumentRevisionCloudService { } } -pub struct DocumentEditorMap { - inner: DashMap>, +#[derive(Clone)] +struct RefCountDocumentHandler(Arc); + +impl RefCountValue for RefCountDocumentHandler { + fn did_remove(&self) { + self.0.close(); + } } -impl DocumentEditorMap { - fn new() -> Self { - Self { inner: DashMap::new() } - } +impl std::ops::Deref for RefCountDocumentHandler { + type Target = Arc; - pub(crate) fn insert(&self, editor_id: &str, editor: Arc) { - if self.inner.contains_key(editor_id) { - log::warn!("Editor:{} already open", editor_id); - } - self.inner.insert(editor_id.to_string(), editor); - } - - pub(crate) fn get(&self, editor_id: &str) -> Option> { - Some(self.inner.get(editor_id)?.clone()) - } - - pub(crate) fn remove(&self, editor_id: &str) { - if let Some(editor) = self.get(editor_id) { - editor.close() - } - self.inner.remove(editor_id); + fn deref(&self) -> &Self::Target { + &self.0 } } #[tracing::instrument(level = "trace", skip(web_socket, handlers))] -fn listen_ws_state_changed(web_socket: Arc, handlers: Arc) { +fn listen_ws_state_changed( + web_socket: Arc, + handlers: Arc>>, +) { tokio::spawn(async move { let mut notify = web_socket.subscribe_state_changed().await; while let Ok(state) = notify.recv().await { - handlers.inner.iter().for_each(|handler| { + handlers.read().await.values().iter().for_each(|handler| { handler.receive_ws_state(&state); }) } diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs index 2c4e5383fb..c9c4c342a0 100644 --- a/frontend/rust-lib/flowy-folder/src/manager.rs +++ b/frontend/rust-lib/flowy-folder/src/manager.rs @@ -168,7 +168,7 @@ impl FolderManager { let pool = self.persistence.db_pool()?; let object_id = folder_id.as_ref(); let disk_cache = SQLiteFolderRevisionPersistence::new(user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(100); + let configuration = RevisionPersistenceConfiguration::new(100, false); let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache, configuration); let rev_compactor = FolderRevisionCompress(); // let history_persistence = SQLiteRevisionHistoryPersistence::new(object_id, pool.clone()); diff --git a/frontend/rust-lib/flowy-folder/tests/workspace/script.rs b/frontend/rust-lib/flowy-folder/tests/workspace/script.rs index 5725258bf2..19e599ff00 100644 --- a/frontend/rust-lib/flowy-folder/tests/workspace/script.rs +++ b/frontend/rust-lib/flowy-folder/tests/workspace/script.rs @@ -70,6 +70,7 @@ pub enum FolderScript { DeleteAllTrash, // Sync + #[allow(dead_code)] AssertCurrentRevId(i64), AssertNextSyncRevId(Option), AssertRevisionState { diff --git a/frontend/rust-lib/flowy-grid/src/manager.rs b/frontend/rust-lib/flowy-grid/src/manager.rs index 68bb61aea9..013b69510f 100644 --- a/frontend/rust-lib/flowy-grid/src/manager.rs +++ b/frontend/rust-lib/flowy-grid/src/manager.rs @@ -1,15 +1,15 @@ use crate::entities::GridLayout; -use crate::services::block_editor::GridBlockRevisionCompress; + use crate::services::grid_editor::{GridRevisionCompress, GridRevisionEditor}; use crate::services::grid_view_manager::make_grid_view_rev_manager; use crate::services::persistence::block_index::BlockIndexCache; use crate::services::persistence::kv::GridKVPersistence; use crate::services::persistence::migration::GridMigration; -use crate::services::persistence::rev_sqlite::{SQLiteGridBlockRevisionPersistence, SQLiteGridRevisionPersistence}; +use crate::services::persistence::rev_sqlite::SQLiteGridRevisionPersistence; use crate::services::persistence::GridDatabase; use crate::services::tasks::GridTaskScheduler; use bytes::Bytes; -use dashmap::DashMap; + use flowy_database::ConnectionPool; use flowy_error::{FlowyError, FlowyResult}; use flowy_grid_data_model::revision::{BuildGridContext, GridRevision, GridViewRevision}; @@ -20,7 +20,8 @@ use flowy_revision::{ use flowy_sync::client_grid::{make_grid_block_operations, make_grid_operations, make_grid_view_operations}; use flowy_sync::entities::revision::Revision; use lib_infra::ref_map::{RefCountHashMap, RefCountValue}; -use std::collections::HashMap; + +use crate::services::block_manager::make_grid_block_rev_manager; use std::sync::Arc; use tokio::sync::RwLock; @@ -92,8 +93,7 @@ impl GridManager { #[tracing::instrument(level = "debug", skip_all, err)] pub async fn create_grid_block>(&self, block_id: T, revisions: Vec) -> FlowyResult<()> { let block_id = block_id.as_ref(); - let db_pool = self.grid_user.db_pool()?; - let rev_manager = self.make_grid_block_rev_manager(block_id, db_pool)?; + let rev_manager = make_grid_block_rev_manager(&self.grid_user, block_id)?; let _ = rev_manager.reset_object(revisions).await?; Ok(()) } @@ -119,13 +119,13 @@ impl GridManager { pub async fn get_grid_editor(&self, grid_id: &str) -> FlowyResult> { match self.grid_editors.read().await.get(grid_id) { None => Err(FlowyError::internal().context("Should call open_grid function first")), - Some(editor) => Ok(editor.clone()), + Some(editor) => Ok(editor), } } async fn get_or_create_grid_editor(&self, grid_id: &str) -> FlowyResult> { if let Some(editor) = self.grid_editors.read().await.get(grid_id) { - return Ok(editor.clone()); + return Ok(editor); } let db_pool = self.grid_user.db_pool()?; @@ -164,29 +164,13 @@ impl GridManager { ) -> FlowyResult>> { let user_id = self.grid_user.user_id()?; let disk_cache = SQLiteGridRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(2); + let configuration = RevisionPersistenceConfiguration::new(2, false); let rev_persistence = RevisionPersistence::new(&user_id, grid_id, disk_cache, configuration); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(grid_id, pool); let rev_compactor = GridRevisionCompress(); let rev_manager = RevisionManager::new(&user_id, grid_id, rev_persistence, rev_compactor, snapshot_persistence); Ok(rev_manager) } - - fn make_grid_block_rev_manager( - &self, - block_id: &str, - pool: Arc, - ) -> FlowyResult>> { - let user_id = self.grid_user.user_id()?; - let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(4); - let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); - let rev_compactor = GridBlockRevisionCompress(); - let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); - let rev_manager = - RevisionManager::new(&user_id, block_id, rev_persistence, rev_compactor, snapshot_persistence); - Ok(rev_manager) - } } pub async fn make_grid_view_data( diff --git a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs index 1750a3473b..474e224fd5 100644 --- a/frontend/rust-lib/flowy-grid/src/services/block_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/block_manager.rs @@ -6,6 +6,7 @@ use crate::services::persistence::block_index::BlockIndexCache; use crate::services::persistence::rev_sqlite::SQLiteGridBlockRevisionPersistence; use crate::services::row::{block_from_row_orders, make_row_from_row_rev, GridBlockSnapshot}; use dashmap::DashMap; +use flowy_database::ConnectionPool; use flowy_error::FlowyResult; use flowy_grid_data_model::revision::{ GridBlockMetaRevision, GridBlockMetaRevisionChangeset, RowChangeset, RowRevision, @@ -46,7 +47,7 @@ impl GridBlockManager { match self.block_editors.get(block_id) { None => { tracing::error!("This is a fatal error, block with id:{} is not exist", block_id); - let editor = Arc::new(make_block_editor(&self.user, block_id).await?); + let editor = Arc::new(make_grid_block_editor(&self.user, block_id).await?); self.block_editors.insert(block_id.to_owned(), editor.clone()); Ok(editor) } @@ -261,24 +262,32 @@ async fn make_block_editors( ) -> FlowyResult>> { let editor_map = DashMap::new(); for block_meta_rev in block_meta_revs { - let editor = make_block_editor(user, &block_meta_rev.block_id).await?; + let editor = make_grid_block_editor(user, &block_meta_rev.block_id).await?; editor_map.insert(block_meta_rev.block_id.clone(), Arc::new(editor)); } Ok(editor_map) } -async fn make_block_editor(user: &Arc, block_id: &str) -> FlowyResult { +async fn make_grid_block_editor(user: &Arc, block_id: &str) -> FlowyResult { tracing::trace!("Open block:{} editor", block_id); let token = user.token()?; let user_id = user.user_id()?; - let pool = user.db_pool()?; + let rev_manager = make_grid_block_rev_manager(user, block_id)?; + GridBlockRevisionEditor::new(&user_id, &token, block_id, rev_manager).await +} +pub fn make_grid_block_rev_manager( + user: &Arc, + block_id: &str, +) -> FlowyResult>> { + let user_id = user.user_id()?; + let pool = user.db_pool()?; let disk_cache = SQLiteGridBlockRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(4); + let configuration = RevisionPersistenceConfiguration::new(4, false); let rev_persistence = RevisionPersistence::new(&user_id, block_id, disk_cache, configuration); let rev_compactor = GridBlockRevisionCompress(); let snapshot_persistence = SQLiteRevisionSnapshotPersistence::new(block_id, pool); let rev_manager = RevisionManager::new(&user_id, block_id, rev_persistence, rev_compactor, snapshot_persistence); - GridBlockRevisionEditor::new(&user_id, &token, block_id, rev_manager).await + Ok(rev_manager) } diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs index 3a9e7ac16c..d88c7ec0e4 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_editor.rs @@ -94,7 +94,13 @@ impl GridRevisionEditor { Ok(editor) } - pub fn close(&self) {} + #[tracing::instrument(name = "close grid editor", level = "trace", skip_all)] + pub fn close(&self) { + let rev_manager = self.rev_manager.clone(); + tokio::spawn(async move { + rev_manager.close().await; + }); + } /// Save the type-option data to disk and send a `GridNotification::DidUpdateField` notification /// to dart side. diff --git a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs index f7de07bf6d..b902d2d25b 100644 --- a/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs +++ b/frontend/rust-lib/flowy-grid/src/services/grid_view_manager.rs @@ -255,7 +255,7 @@ pub async fn make_grid_view_rev_manager( let pool = user.db_pool()?; let disk_cache = SQLiteGridViewRevisionPersistence::new(&user_id, pool.clone()); - let configuration = RevisionPersistenceConfiguration::new(2); + let configuration = RevisionPersistenceConfiguration::new(2, false); let rev_persistence = RevisionPersistence::new(&user_id, view_id, disk_cache, configuration); let rev_compactor = GridViewRevisionCompress(); diff --git a/frontend/rust-lib/flowy-grid/src/services/mod.rs b/frontend/rust-lib/flowy-grid/src/services/mod.rs index ab864c544a..1e759a082a 100644 --- a/frontend/rust-lib/flowy-grid/src/services/mod.rs +++ b/frontend/rust-lib/flowy-grid/src/services/mod.rs @@ -1,7 +1,7 @@ mod util; pub mod block_editor; -mod block_manager; +pub mod block_manager; mod block_manager_trait_impl; pub mod cell; pub mod field; diff --git a/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs b/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs index a4ebe36d2e..e88cd9fd9d 100644 --- a/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs +++ b/frontend/rust-lib/flowy-grid/src/services/tasks/scheduler.rs @@ -1,4 +1,4 @@ -use crate::services::tasks::queue::{GridTaskQueue, TaskHandlerId}; +use crate::services::tasks::queue::GridTaskQueue; use crate::services::tasks::runner::GridTaskRunner; use crate::services::tasks::store::GridTaskStore; use crate::services::tasks::task::Task; @@ -7,22 +7,28 @@ use crate::services::tasks::{TaskContent, TaskId, TaskStatus}; use flowy_error::FlowyError; use lib_infra::future::BoxResultFuture; use lib_infra::ref_map::{RefCountHashMap, RefCountValue}; -use std::collections::HashMap; + use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, RwLock}; -pub(crate) trait GridTaskHandler: Send + Sync + 'static + RefCountValue { +pub(crate) trait GridTaskHandler: Send + Sync + 'static { fn handler_id(&self) -> &str; fn process_content(&self, content: TaskContent) -> BoxResultFuture<(), FlowyError>; } +#[derive(Clone)] +struct RefCountTaskHandler(Arc); +impl RefCountValue for RefCountTaskHandler { + fn did_remove(&self) {} +} + pub struct GridTaskScheduler { queue: GridTaskQueue, store: GridTaskStore, notifier: watch::Sender, - handlers: RefCountHashMap>, + handlers: RefCountHashMap, } impl GridTaskScheduler { @@ -51,7 +57,7 @@ impl GridTaskScheduler { T: GridTaskHandler, { let handler_id = handler.handler_id().to_owned(); - self.handlers.insert(handler_id, handler); + self.handlers.insert(handler_id, RefCountTaskHandler(handler)); } pub(crate) fn unregister_handler>(&mut self, handler_id: T) { @@ -74,7 +80,7 @@ impl GridTaskScheduler { let content = task.content.take()?; task.set_status(TaskStatus::Processing); - let _ = match handler.process_content(content).await { + let _ = match handler.0.process_content(content).await { Ok(_) => { task.set_status(TaskStatus::Done); let _ = ret.send(task.into()); diff --git a/frontend/rust-lib/flowy-revision/src/cache/reset.rs b/frontend/rust-lib/flowy-revision/src/cache/reset.rs index 4c4c223818..8fa522c033 100644 --- a/frontend/rust-lib/flowy-revision/src/cache/reset.rs +++ b/frontend/rust-lib/flowy-revision/src/cache/reset.rs @@ -60,7 +60,7 @@ where } async fn reset_object(&self) -> FlowyResult<()> { - let configuration = RevisionPersistenceConfiguration::new(2); + let configuration = RevisionPersistenceConfiguration::new(2, false); let rev_persistence = Arc::new(RevisionPersistence::from_disk_cache( &self.user_id, self.target.target_id(), diff --git a/frontend/rust-lib/flowy-revision/src/rev_manager.rs b/frontend/rust-lib/flowy-revision/src/rev_manager.rs index c1bd8f531b..cb613bad65 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_manager.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_manager.rs @@ -123,6 +123,10 @@ impl RevisionManager { B::deserialize_revisions(&self.object_id, revisions) } + pub async fn close(&self) { + let _ = self.rev_persistence.compact_lagging_revisions(&self.rev_compress).await; + } + pub async fn load_revisions(&self) -> FlowyResult> { let revisions = RevisionLoader { object_id: self.object_id.clone(), diff --git a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs index 7b8f4c2b9e..901a0b214f 100644 --- a/frontend/rust-lib/flowy-revision/src/rev_persistence.rs +++ b/frontend/rust-lib/flowy-revision/src/rev_persistence.rs @@ -17,22 +17,32 @@ pub const REVISION_WRITE_INTERVAL_IN_MILLIS: u64 = 600; #[derive(Clone)] pub struct RevisionPersistenceConfiguration { merge_threshold: usize, + merge_lagging: bool, } impl RevisionPersistenceConfiguration { - pub fn new(merge_threshold: usize) -> Self { + pub fn new(merge_threshold: usize, merge_lagging: bool) -> Self { debug_assert!(merge_threshold > 1); if merge_threshold > 1 { - Self { merge_threshold } + Self { + merge_threshold, + merge_lagging, + } } else { - Self { merge_threshold: 100 } + Self { + merge_threshold: 100, + merge_lagging, + } } } } impl std::default::Default for RevisionPersistenceConfiguration { fn default() -> Self { - Self { merge_threshold: 100 } + Self { + merge_threshold: 100, + merge_lagging: false, + } } } @@ -98,6 +108,36 @@ where Ok(()) } + #[tracing::instrument(level = "trace", skip_all, err)] + pub async fn compact_lagging_revisions<'a>( + &'a self, + rev_compress: &Arc, + ) -> FlowyResult<()> { + if !self.configuration.merge_lagging { + return Ok(()); + } + + let mut sync_seq = self.sync_seq.write().await; + let compact_seq = sync_seq.compact(); + if !compact_seq.is_empty() { + let range = RevisionRange { + start: *compact_seq.front().unwrap(), + end: *compact_seq.back().unwrap(), + }; + + let revisions = self.revisions_in_range(&range).await?; + debug_assert_eq!(range.len() as usize, revisions.len()); + // compact multiple revisions into one + let merged_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?; + tracing::Span::current().record("rev_id", &merged_revision.rev_id); + let _ = sync_seq.recv(merged_revision.rev_id)?; + + // replace the revisions in range with compact revision + self.compact(&range, merged_revision).await?; + } + Ok(()) + } + /// Save the revision to disk and append it to the end of the sync sequence. #[tracing::instrument(level = "trace", skip_all, fields(rev_id, compact_range, object_id=%self.object_id), err)] pub(crate) async fn add_sync_revision<'a>( @@ -108,7 +148,7 @@ where let mut sync_seq = self.sync_seq.write().await; let compact_length = sync_seq.compact_length; - // Before the new_revision is pushed into the sync_seq, we check if the current `step` of the + // Before the new_revision is pushed into the sync_seq, we check if the current `compact_length` of the // sync_seq is less equal to or greater than the merge threshold. If yes, it's needs to merged // with the new_revision into one revision. let mut compact_seq = VecDeque::default(); diff --git a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs index 76f1833554..864002051b 100644 --- a/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs +++ b/frontend/rust-lib/flowy-revision/tests/revision_test/script.rs @@ -63,7 +63,7 @@ impl RevisionTest { pub async fn new_with_configuration(merge_threshold: i64) -> Self { let user_id = nanoid!(10); let object_id = nanoid!(6); - let configuration = RevisionPersistenceConfiguration::new(merge_threshold as usize); + let configuration = RevisionPersistenceConfiguration::new(merge_threshold as usize, false); let disk_cache = RevisionDiskCacheMock::new(vec![]); let persistence = RevisionPersistence::new(&user_id, &object_id, disk_cache, configuration.clone()); let compress = RevisionCompressMock {}; diff --git a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs index 1fa6804177..35def846a7 100644 --- a/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs +++ b/frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs @@ -165,7 +165,7 @@ impl ViewDataProcessor for DocumentViewDataProcessor { let manager = self.0.clone(); let view_id = view_id.to_string(); FutureResult::new(async move { - let _ = manager.close_document_editor(view_id)?; + let _ = manager.close_document_editor(view_id).await?; Ok(()) }) } diff --git a/shared-lib/lib-infra/src/ref_map.rs b/shared-lib/lib-infra/src/ref_map.rs index 1b9e3baad7..8ef4ba8a6b 100644 --- a/shared-lib/lib-infra/src/ref_map.rs +++ b/shared-lib/lib-infra/src/ref_map.rs @@ -22,16 +22,26 @@ impl RefCountHandler { pub struct RefCountHashMap(HashMap>); +impl std::default::Default for RefCountHashMap { + fn default() -> Self { + Self(HashMap::new()) + } +} + impl RefCountHashMap where T: Clone + Send + Sync + RefCountValue, { pub fn new() -> Self { - Self(Default::default()) + Self::default() } pub fn get(&self, key: &str) -> Option { - self.0.get(key).and_then(|handler| Some(handler.inner.clone())) + self.0.get(key).map(|handler| handler.inner.clone()) + } + + pub fn values(&self) -> Vec { + self.0.values().map(|value| value.inner.clone()).collect::>() } pub fn insert(&mut self, key: String, value: T) {