Refactor/revision compose (#1410)

frontend/rust-lib/flowy-folder/src/manager.rs

@@ -12,16 +12,15 @@ use crate::{
     },
 };
 use bytes::Bytes;
+use flowy_document::editor::initial_read_me;
 use flowy_error::FlowyError;
 use flowy_folder_data_model::user_default;
-use flowy_revision::disk::SQLiteDeltaDocumentRevisionPersistence;
 use flowy_revision::{RevisionManager, RevisionPersistence, RevisionWebSocket, SQLiteRevisionSnapshotPersistence};
-
-use flowy_document::editor::initial_read_me;
 use flowy_sync::{client_folder::FolderPad, entities::ws_data::ServerRevisionWSData};
 use lazy_static::lazy_static;
 use lib_infra::future::FutureResult;
 
+use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence;
 use std::{collections::HashMap, convert::TryInto, fmt::Formatter, sync::Arc};
 use tokio::sync::RwLock as TokioRwLock;
 lazy_static! {
@@ -165,7 +164,7 @@ impl FolderManager {
 
         let pool = self.persistence.db_pool()?;
         let object_id = folder_id.as_ref();
-        let disk_cache = SQLiteDeltaDocumentRevisionPersistence::new(user_id, pool.clone());
+        let disk_cache = SQLiteFolderRevisionPersistence::new(user_id, pool.clone());
         let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache);
         let rev_compactor = FolderRevisionCompress();
         // let history_persistence = SQLiteRevisionHistoryPersistence::new(object_id, pool.clone());

frontend/rust-lib/flowy-folder/src/services/folder_editor.rs

@@ -12,6 +12,7 @@ use flowy_sync::{
 };
 use lib_infra::future::FutureResult;
 
+use flowy_database::ConnectionPool;
 use lib_ot::core::EmptyAttributes;
 use parking_lot::RwLock;
 use std::sync::Arc;
@@ -21,7 +22,7 @@ pub struct FolderEditor {
     #[allow(dead_code)]
     pub(crate) folder_id: FolderId,
     pub(crate) folder: Arc<RwLock<FolderPad>>,
-    rev_manager: Arc<RevisionManager>,
+    rev_manager: Arc<RevisionManager<Arc<ConnectionPool>>>,
     #[cfg(feature = "sync")]
     ws_manager: Arc<flowy_revision::RevisionWebSocketManager>,
 }
@@ -32,7 +33,7 @@ impl FolderEditor {
         user_id: &str,
         folder_id: &FolderId,
         token: &str,
-        mut rev_manager: RevisionManager,
+        mut rev_manager: RevisionManager<Arc<ConnectionPool>>,
         web_socket: Arc<dyn RevisionWebSocket>,
     ) -> FlowyResult<Self> {
         let cloud = Arc::new(FolderRevisionCloudService {
@@ -139,7 +140,7 @@ impl RevisionCloudService for FolderRevisionCloudService {
 
 #[cfg(feature = "flowy_unit_test")]
 impl FolderEditor {
-    pub fn rev_manager(&self) -> Arc<RevisionManager> {
+    pub fn rev_manager(&self) -> Arc<RevisionManager<Arc<ConnectionPool>>> {
         self.rev_manager.clone()
     }
 }

frontend/rust-lib/flowy-folder/src/services/persistence/migration.rs

@@ -7,13 +7,13 @@ use bytes::Bytes;
 use flowy_database::kv::KV;
 use flowy_error::{FlowyError, FlowyResult};
 use flowy_folder_data_model::revision::{AppRevision, FolderRevision, ViewRevision, WorkspaceRevision};
-use flowy_revision::disk::SQLiteDeltaDocumentRevisionPersistence;
 use flowy_revision::reset::{RevisionResettable, RevisionStructReset};
 use flowy_sync::client_folder::make_folder_rev_json_str;
 use flowy_sync::entities::revision::Revision;
 use flowy_sync::server_folder::FolderOperationsBuilder;
 use flowy_sync::{client_folder::FolderPad, entities::revision::md5};
 
+use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence;
 use std::sync::Arc;
 
 const V1_MIGRATION: &str = "FOLDER_V1_MIGRATION";
@@ -113,7 +113,7 @@ impl FolderMigration {
         };
 
         let pool = self.database.db_pool()?;
-        let disk_cache = SQLiteDeltaDocumentRevisionPersistence::new(&self.user_id, pool);
+        let disk_cache = SQLiteFolderRevisionPersistence::new(&self.user_id, pool);
         let reset = RevisionStructReset::new(&self.user_id, object, Arc::new(disk_cache));
         reset.run().await
     }
@@ -144,4 +144,12 @@ impl RevisionResettable for FolderRevisionResettable {
         let json = make_folder_rev_json_str(&folder)?;
         Ok(json)
     }
+
+    fn read_record(&self) -> Option<String> {
+        KV::get_str(self.target_id())
+    }
+
+    fn set_record(&self, record: String) {
+        KV::set_str(self.target_id(), record);
+    }
 }

frontend/rust-lib/flowy-folder/src/services/persistence/mod.rs

@@ -1,4 +1,5 @@
 mod migration;
+pub mod rev_sqlite;
 pub mod version_1;
 mod version_2;
 
@@ -10,10 +11,10 @@ use crate::{
 use flowy_database::ConnectionPool;
 use flowy_error::{FlowyError, FlowyResult};
 use flowy_folder_data_model::revision::{AppRevision, TrashRevision, ViewRevision, WorkspaceRevision};
-use flowy_revision::disk::{RevisionRecord, RevisionState};
-use flowy_revision::mk_text_block_revision_disk_cache;
+use flowy_revision::disk::{RevisionDiskCache, RevisionRecord, RevisionState};
 use flowy_sync::{client_folder::FolderPad, entities::revision::Revision};
 
+use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence;
 use flowy_sync::server_folder::FolderOperationsBuilder;
 use std::sync::Arc;
 use tokio::sync::RwLock;
@@ -121,3 +122,10 @@ impl FolderPersistence {
         disk_cache.delete_and_insert_records(folder_id.as_ref(), None, vec![record])
     }
 }
+
+pub fn mk_text_block_revision_disk_cache(
+    user_id: &str,
+    pool: Arc<ConnectionPool>,
+) -> Arc<dyn RevisionDiskCache<Arc<ConnectionPool>, Error = FlowyError>> {
+    Arc::new(SQLiteFolderRevisionPersistence::new(user_id, pool))
+}

frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_rev_sqlite.rs

@@ -0,0 +1,284 @@
+use bytes::Bytes;
+use diesel::{sql_types::Integer, update, SqliteConnection};
+use flowy_database::{
+    impl_sql_integer_expression, insert_or_ignore_into,
+    prelude::*,
+    schema::{rev_table, rev_table::dsl},
+    ConnectionPool,
+};
+use flowy_error::{internal_error, FlowyError, FlowyResult};
+use flowy_revision::disk::{RevisionChangeset, RevisionDiskCache, RevisionRecord, RevisionState};
+use flowy_sync::{
+    entities::revision::{RevType, Revision, RevisionRange},
+    util::md5,
+};
+
+use std::sync::Arc;
+
+pub struct SQLiteFolderRevisionPersistence {
+    user_id: String,
+    pub(crate) pool: Arc<ConnectionPool>,
+}
+
+impl RevisionDiskCache<Arc<ConnectionPool>> for SQLiteFolderRevisionPersistence {
+    type Error = FlowyError;
+
+    fn create_revision_records(&self, revision_records: Vec<RevisionRecord>) -> Result<(), Self::Error> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        let _ = FolderRevisionSql::create(revision_records, &*conn)?;
+        Ok(())
+    }
+
+    fn get_connection(&self) -> Result<Arc<ConnectionPool>, Self::Error> {
+        Ok(self.pool.clone())
+    }
+
+    fn read_revision_records(
+        &self,
+        object_id: &str,
+        rev_ids: Option<Vec<i64>>,
+    ) -> Result<Vec<RevisionRecord>, Self::Error> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        let records = FolderRevisionSql::read(&self.user_id, object_id, rev_ids, &*conn)?;
+        Ok(records)
+    }
+
+    fn read_revision_records_with_range(
+        &self,
+        object_id: &str,
+        range: &RevisionRange,
+    ) -> Result<Vec<RevisionRecord>, Self::Error> {
+        let conn = &*self.pool.get().map_err(internal_error)?;
+        let revisions = FolderRevisionSql::read_with_range(&self.user_id, object_id, range.clone(), conn)?;
+        Ok(revisions)
+    }
+
+    fn update_revision_record(&self, changesets: Vec<RevisionChangeset>) -> FlowyResult<()> {
+        let conn = &*self.pool.get().map_err(internal_error)?;
+        let _ = conn.immediate_transaction::<_, FlowyError, _>(|| {
+            for changeset in changesets {
+                let _ = FolderRevisionSql::update(changeset, conn)?;
+            }
+            Ok(())
+        })?;
+        Ok(())
+    }
+
+    fn delete_revision_records(&self, object_id: &str, rev_ids: Option<Vec<i64>>) -> Result<(), Self::Error> {
+        let conn = &*self.pool.get().map_err(internal_error)?;
+        let _ = FolderRevisionSql::delete(object_id, rev_ids, conn)?;
+        Ok(())
+    }
+
+    fn delete_and_insert_records(
+        &self,
+        object_id: &str,
+        deleted_rev_ids: Option<Vec<i64>>,
+        inserted_records: Vec<RevisionRecord>,
+    ) -> Result<(), Self::Error> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        conn.immediate_transaction::<_, FlowyError, _>(|| {
+            let _ = FolderRevisionSql::delete(object_id, deleted_rev_ids, &*conn)?;
+            let _ = FolderRevisionSql::create(inserted_records, &*conn)?;
+            Ok(())
+        })
+    }
+}
+
+impl SQLiteFolderRevisionPersistence {
+    pub fn new(user_id: &str, pool: Arc<ConnectionPool>) -> Self {
+        Self {
+            user_id: user_id.to_owned(),
+            pool,
+        }
+    }
+}
+
+struct FolderRevisionSql {}
+
+impl FolderRevisionSql {
+    fn create(revision_records: Vec<RevisionRecord>, conn: &SqliteConnection) -> Result<(), FlowyError> {
+        // Batch insert: https://diesel.rs/guides/all-about-inserts.html
+
+        let records = revision_records
+            .into_iter()
+            .map(|record| {
+                tracing::trace!(
+                    "[TextRevisionSql] create revision: {}:{:?}",
+                    record.revision.object_id,
+                    record.revision.rev_id
+                );
+                let rev_state: TextRevisionState = record.state.into();
+                (
+                    dsl::doc_id.eq(record.revision.object_id),
+                    dsl::base_rev_id.eq(record.revision.base_rev_id),
+                    dsl::rev_id.eq(record.revision.rev_id),
+                    dsl::data.eq(record.revision.bytes),
+                    dsl::state.eq(rev_state),
+                    dsl::ty.eq(RevTableType::Local),
+                )
+            })
+            .collect::<Vec<_>>();
+
+        let _ = insert_or_ignore_into(dsl::rev_table).values(&records).execute(conn)?;
+        Ok(())
+    }
+
+    fn update(changeset: RevisionChangeset, conn: &SqliteConnection) -> Result<(), FlowyError> {
+        let state: TextRevisionState = changeset.state.clone().into();
+        let filter = dsl::rev_table
+            .filter(dsl::rev_id.eq(changeset.rev_id.as_ref()))
+            .filter(dsl::doc_id.eq(changeset.object_id));
+        let _ = update(filter).set(dsl::state.eq(state)).execute(conn)?;
+        tracing::debug!(
+            "[TextRevisionSql] update revision:{} state:to {:?}",
+            changeset.rev_id,
+            changeset.state
+        );
+        Ok(())
+    }
+
+    fn read(
+        user_id: &str,
+        object_id: &str,
+        rev_ids: Option<Vec<i64>>,
+        conn: &SqliteConnection,
+    ) -> Result<Vec<RevisionRecord>, FlowyError> {
+        let mut sql = dsl::rev_table.filter(dsl::doc_id.eq(object_id)).into_boxed();
+        if let Some(rev_ids) = rev_ids {
+            sql = sql.filter(dsl::rev_id.eq_any(rev_ids));
+        }
+        let rows = sql.order(dsl::rev_id.asc()).load::<RevisionTable>(conn)?;
+        let records = rows
+            .into_iter()
+            .map(|row| mk_revision_record_from_table(user_id, row))
+            .collect::<Vec<_>>();
+
+        Ok(records)
+    }
+
+    fn read_with_range(
+        user_id: &str,
+        object_id: &str,
+        range: RevisionRange,
+        conn: &SqliteConnection,
+    ) -> Result<Vec<RevisionRecord>, FlowyError> {
+        let rev_tables = dsl::rev_table
+            .filter(dsl::rev_id.ge(range.start))
+            .filter(dsl::rev_id.le(range.end))
+            .filter(dsl::doc_id.eq(object_id))
+            .order(dsl::rev_id.asc())
+            .load::<RevisionTable>(conn)?;
+
+        let revisions = rev_tables
+            .into_iter()
+            .map(|table| mk_revision_record_from_table(user_id, table))
+            .collect::<Vec<_>>();
+        Ok(revisions)
+    }
+
+    fn delete(object_id: &str, rev_ids: Option<Vec<i64>>, conn: &SqliteConnection) -> Result<(), FlowyError> {
+        let mut sql = diesel::delete(dsl::rev_table).into_boxed();
+        sql = sql.filter(dsl::doc_id.eq(object_id));
+
+        if let Some(rev_ids) = rev_ids {
+            tracing::trace!("[TextRevisionSql] Delete revision: {}:{:?}", object_id, rev_ids);
+            sql = sql.filter(dsl::rev_id.eq_any(rev_ids));
+        }
+
+        let affected_row = sql.execute(conn)?;
+        tracing::trace!("[TextRevisionSql] Delete {} rows", affected_row);
+        Ok(())
+    }
+}
+
+#[derive(PartialEq, Clone, Debug, Queryable, Identifiable, Insertable, Associations)]
+#[table_name = "rev_table"]
+struct RevisionTable {
+    id: i32,
+    doc_id: String,
+    base_rev_id: i64,
+    rev_id: i64,
+    data: Vec<u8>,
+    state: TextRevisionState,
+    ty: RevTableType, // Deprecated
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, FromSqlRow, AsExpression)]
+#[repr(i32)]
+#[sql_type = "Integer"]
+enum TextRevisionState {
+    Sync = 0,
+    Ack = 1,
+}
+impl_sql_integer_expression!(TextRevisionState);
+impl_rev_state_map!(TextRevisionState);
+
+impl std::default::Default for TextRevisionState {
+    fn default() -> Self {
+        TextRevisionState::Sync
+    }
+}
+
+fn mk_revision_record_from_table(user_id: &str, table: RevisionTable) -> RevisionRecord {
+    let md5 = md5(&table.data);
+    let revision = Revision::new(
+        &table.doc_id,
+        table.base_rev_id,
+        table.rev_id,
+        Bytes::from(table.data),
+        user_id,
+        md5,
+    );
+    RevisionRecord {
+        revision,
+        state: table.state.into(),
+        write_to_disk: false,
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, FromSqlRow, AsExpression)]
+#[repr(i32)]
+#[sql_type = "Integer"]
+pub enum RevTableType {
+    Local = 0,
+    Remote = 1,
+}
+impl_sql_integer_expression!(RevTableType);
+
+impl std::default::Default for RevTableType {
+    fn default() -> Self {
+        RevTableType::Local
+    }
+}
+
+impl std::convert::From<i32> for RevTableType {
+    fn from(value: i32) -> Self {
+        match value {
+            0 => RevTableType::Local,
+            1 => RevTableType::Remote,
+            o => {
+                tracing::error!("Unsupported rev type {}, fallback to RevTableType::Local", o);
+                RevTableType::Local
+            }
+        }
+    }
+}
+
+impl std::convert::From<RevType> for RevTableType {
+    fn from(ty: RevType) -> Self {
+        match ty {
+            RevType::DeprecatedLocal => RevTableType::Local,
+            RevType::DeprecatedRemote => RevTableType::Remote,
+        }
+    }
+}
+
+impl std::convert::From<RevTableType> for RevType {
+    fn from(ty: RevTableType) -> Self {
+        match ty {
+            RevTableType::Local => RevType::DeprecatedLocal,
+            RevTableType::Remote => RevType::DeprecatedRemote,
+        }
+    }
+}

frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/mod.rs

@@ -0,0 +1,2 @@
+mod folder_rev_sqlite;
+pub use folder_rev_sqlite::*;

frontend/rust-lib/flowy-folder/src/services/web_socket.rs

@@ -1,5 +1,6 @@
 use crate::services::FOLDER_SYNC_INTERVAL_IN_MILLIS;
 use bytes::Bytes;
+use flowy_database::ConnectionPool;
 use flowy_error::{FlowyError, FlowyResult};
 use flowy_revision::*;
 use flowy_sync::entities::revision::Revision;
@@ -37,13 +38,13 @@ impl FolderResolveOperations {
     }
 }
 
-pub type FolderConflictController = ConflictController<FolderResolveOperations>;
+pub type FolderConflictController = ConflictController<FolderResolveOperations, Arc<ConnectionPool>>;
 
 #[allow(dead_code)]
 pub(crate) async fn make_folder_ws_manager(
     user_id: &str,
     folder_id: &str,
-    rev_manager: Arc<RevisionManager>,
+    rev_manager: Arc<RevisionManager<Arc<ConnectionPool>>>,
     web_socket: Arc<dyn RevisionWebSocket>,
     folder_pad: Arc<RwLock<FolderPad>>,
 ) -> Arc<RevisionWebSocketManager> {