chore: using zstd as collab_db compression method and Deflated as zip file compression method (#4318)

frontend/appflowy_tauri/src-tauri/Cargo.lock

@@ -780,7 +780,7 @@ dependencies = [
 [[package]]
 name = "collab"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -800,7 +800,7 @@ dependencies = [
 [[package]]
 name = "collab-database"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -827,7 +827,7 @@ dependencies = [
 [[package]]
 name = "collab-document"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "collab",
@@ -844,7 +844,7 @@ dependencies = [
 [[package]]
 name = "collab-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "bytes",
@@ -858,7 +858,7 @@ dependencies = [
 [[package]]
 name = "collab-folder"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "chrono",
@@ -893,7 +893,7 @@ dependencies = [
 [[package]]
 name = "collab-plugins"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -923,7 +923,7 @@ dependencies = [
 [[package]]
 name = "collab-user"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "collab",

frontend/appflowy_tauri/src-tauri/Cargo.toml

@@ -67,13 +67,13 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "215
 # To switch to the local path, run:
 # scripts/tool/update_collab_source.sh
 # ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
 
 
 

frontend/rust-lib/Cargo.lock

@@ -686,7 +686,7 @@ dependencies = [
 [[package]]
 name = "collab"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -706,7 +706,7 @@ dependencies = [
 [[package]]
 name = "collab-database"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -733,7 +733,7 @@ dependencies = [
 [[package]]
 name = "collab-document"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "collab",
@@ -750,7 +750,7 @@ dependencies = [
 [[package]]
 name = "collab-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "bytes",
@@ -764,7 +764,7 @@ dependencies = [
 [[package]]
 name = "collab-folder"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "chrono",
@@ -799,7 +799,7 @@ dependencies = [
 [[package]]
 name = "collab-plugins"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -829,7 +829,7 @@ dependencies = [
 [[package]]
 name = "collab-user"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
  "anyhow",
  "collab",

frontend/rust-lib/Cargo.toml

@@ -117,10 +117,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "215
 # To switch to the local path, run:
 # scripts/tool/update_collab_source.sh
 # ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }

frontend/rust-lib/collab-integrate/src/collab_builder.rs

@@ -194,7 +194,7 @@ impl AppFlowyCollabBuilder {
         object_type,
         collab_db,
         collab_doc_state,
-        &persistence_config,
+        persistence_config,
         build_config,
       )
       .await
@@ -222,7 +222,7 @@ impl AppFlowyCollabBuilder {
     object_type: CollabType,
     collab_db: Weak<CollabKVDB>,
     collab_doc_state: CollabDocState,
-    persistence_config: &CollabPersistenceConfig,
+    persistence_config: CollabPersistenceConfig,
     build_config: CollabBuilderConfig,
   ) -> Result<Arc<MutexCollab>, Error> {
     let collab = Arc::new(
@@ -232,7 +232,6 @@ impl AppFlowyCollabBuilder {
           uid,
           collab_db.clone(),
           persistence_config.clone(),
-          self.rocksdb_backup.lock().clone(),
         ))
         .with_device_id(self.device_id.clone())
         .build()?,

frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs

@@ -1,8 +1,9 @@
-use std::sync::Weak;
+use collab_entity::CollabType;
-
-use diesel::SqliteConnection;
-
 use collab_integrate::{CollabSnapshot, PersistenceError, SnapshotPersistence};
+use diesel::dsl::count_star;
+use diesel::internal::derives::multiconnection::chrono;
+use diesel::internal::derives::multiconnection::chrono::Local;
+use diesel::SqliteConnection;
 use flowy_error::FlowyError;
 use flowy_sqlite::{
   insert_or_ignore_into,
@@ -11,6 +12,7 @@ use flowy_sqlite::{
 };
 use flowy_user::manager::UserManager;
 use lib_infra::util::timestamp;
+use std::sync::Weak;
 
 pub struct SnapshotDBImpl(pub Weak<UserManager>);
 
@@ -33,9 +35,10 @@ impl SnapshotPersistence for SnapshotDBImpl {
     &self,
     uid: i64,
     object_id: &str,
-    title: String,
+    collab_type: &CollabType,
-    snapshot_data: Vec<u8>,
+    encoded_v1: Vec<u8>,
   ) -> Result<(), PersistenceError> {
+    let collab_type = collab_type.clone();
     let object_id = object_id.to_string();
     let weak_user_session = self.0.clone();
     tokio::task::spawn_blocking(move || {
@@ -49,19 +52,10 @@ impl SnapshotPersistence for SnapshotDBImpl {
 
         // Save the snapshot data to disk
         let result = CollabSnapshotTableSql::create(
-          CollabSnapshotRow {
+          CollabSnapshotRow::new(object_id.clone(), collab_type.to_string(), encoded_v1),
-            id: uuid::Uuid::new_v4().to_string(),
-            object_id: object_id.clone(),
-            title,
-            desc: "".to_string(),
-            collab_type: "".to_string(),
-            timestamp: timestamp(),
-            data: snapshot_data,
-          },
           &mut conn,
         )
         .map_err(|e| PersistenceError::Internal(e.into()));
-
         if let Err(e) = result {
           tracing::warn!("create snapshot error: {:?}", e);
         }
@@ -84,6 +78,20 @@ struct CollabSnapshotRow {
   data: Vec<u8>,
 }
 
+impl CollabSnapshotRow {
+  pub fn new(object_id: String, collab_type: String, data: Vec<u8>) -> Self {
+    Self {
+      id: uuid::Uuid::new_v4().to_string(),
+      object_id,
+      title: "".to_string(),
+      desc: "".to_string(),
+      collab_type,
+      timestamp: timestamp(),
+      data,
+    }
+  }
+}
+
 impl From<CollabSnapshotRow> for CollabSnapshot {
   fn from(table: CollabSnapshotRow) -> Self {
     Self {
@@ -96,7 +104,6 @@ impl From<CollabSnapshotRow> for CollabSnapshot {
 struct CollabSnapshotTableSql;
 impl CollabSnapshotTableSql {
   fn create(row: CollabSnapshotRow, conn: &mut SqliteConnection) -> Result<(), FlowyError> {
-    // Batch insert: https://diesel.rs/guides/all-about-inserts.html
     let values = (
       dsl::id.eq(row.id),
       dsl::object_id.eq(row.object_id),
@@ -109,6 +116,16 @@ impl CollabSnapshotTableSql {
     let _ = insert_or_ignore_into(dsl::collab_snapshot)
       .values(values)
       .execute(conn)?;
+
+    // Count the total number of rows
+    // If there are more than 10 rows, delete snapshots older than 5 days
+    let total_rows: i64 = dsl::collab_snapshot.select(count_star()).first(conn)?;
+    if total_rows > 10 {
+      let five_days_ago = Local::now() - chrono::Duration::days(5);
+      let _ =
+        diesel::delete(dsl::collab_snapshot.filter(dsl::timestamp.lt(five_days_ago.timestamp())))
+          .execute(conn)?;
+    };
     Ok(())
   }
 

frontend/rust-lib/flowy-database2/src/manager.rs

@@ -133,7 +133,7 @@ impl DatabaseManager {
       CollabType::WorkspaceDatabase,
       collab_db.clone(),
       collab_raw_data,
-      &config,
+      config.clone(),
     );
     let workspace_database =
       WorkspaceDatabase::open(uid, collab, collab_db, config, collab_builder);
@@ -461,7 +461,7 @@ impl DatabaseCollabService for UserDatabaseCollabServiceImpl {
     object_type: CollabType,
     collab_db: Weak<CollabKVDB>,
     collab_raw_data: CollabDocState,
-    config: &CollabPersistenceConfig,
+    persistence_config: CollabPersistenceConfig,
   ) -> Arc<MutexCollab> {
     block_on(self.collab_builder.build_with_config(
       uid,
@@ -469,7 +469,7 @@ impl DatabaseCollabService for UserDatabaseCollabServiceImpl {
       object_type,
       collab_db,
       collab_raw_data,
-      config,
+      persistence_config,
       CollabBuilderConfig::default().sync_enable(true),
     ))
     .unwrap()

frontend/rust-lib/flowy-document/src/entities.rs

@@ -398,9 +398,6 @@ pub struct DocumentSnapshotPB {
 
   #[pb(index = 3)]
   pub created_at: i64,
-
-  #[pb(index = 4)]
-  pub data: Vec<u8>,
 }
 
 #[derive(Debug, Default, ProtoBuf)]

frontend/rust-lib/flowy-document/src/manager.rs

@@ -15,7 +15,7 @@ use parking_lot::Mutex;
 use tracing::{event, instrument};
 
 use collab_integrate::collab_builder::{AppFlowyCollabBuilder, CollabBuilderConfig};
-use collab_integrate::{CollabKVAction, CollabKVDB};
+use collab_integrate::{CollabKVAction, CollabKVDB, CollabPersistenceConfig};
 use flowy_document_deps::cloud::DocumentCloudService;
 use flowy_error::{internal_error, ErrorCode, FlowyError, FlowyResult};
 use flowy_storage::FileStorageService;
@@ -219,7 +219,6 @@ impl DocumentManager {
         snapshot_id: snapshot.snapshot_id,
         snapshot_desc: "".to_string(),
         created_at: snapshot.created_at,
-        data: snapshot.data,
       })
       .collect::<Vec<_>>();
 
@@ -236,12 +235,13 @@ impl DocumentManager {
     let db = self.user.collab_db(uid)?;
     let collab = self
       .collab_builder
-      .build(
+      .build_with_config(
         uid,
         doc_id,
         CollabType::Document,
-        doc_state,
         db,
+        doc_state,
+        CollabPersistenceConfig::default().snapshot_per_update(100),
         CollabBuilderConfig::default().sync_enable(sync_enable),
       )
       .await?;

frontend/rust-lib/flowy-folder/src/manager.rs

@@ -147,7 +147,9 @@ impl FolderManager {
         CollabType::Folder,
         collab_db,
         collab_doc_state,
-        &CollabPersistenceConfig::new().enable_snapshot(true),
+        CollabPersistenceConfig::new()
+          .enable_snapshot(true)
+          .snapshot_per_update(20),
         CollabBuilderConfig::default().sync_enable(true),
       )
       .await?;

frontend/rust-lib/lib-infra/Cargo.toml

@@ -21,4 +21,4 @@ tempfile = "3.8.1"
 validator = "0.16.0"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-zip = "0.6.6"
+zip = { version = "0.6.6", features = ["deflate"] }

frontend/rust-lib/lib-infra/src/file_util.rs

@@ -79,7 +79,7 @@ pub fn zip_folder(src_path: impl AsRef<Path>, dest_path: &Path) -> io::Result<()
 
   let file = File::create(dest_path)?;
   let mut zip = ZipWriter::new(file);
-  let options = FileOptions::default().compression_method(zip::CompressionMethod::Stored);
+  let options = FileOptions::default().compression_method(zip::CompressionMethod::Deflated);
 
   for entry in WalkDir::new(&src_path) {
     let entry = entry?;