chore: using zstd as collab_db compression method and Deflated as zip file compression method (#4318)

2024-08-30 18:12:39 +00:00 · 2024-01-06 12:46:11 +08:00 · 2024-01-06 12:46:11 +08:00 · 2557e4f3cc
commit 2557e4f3cc
parent 79c912219d
12 changed files with 76 additions and 61 deletions
--- a/frontend/appflowy_tauri/src-tauri/Cargo.lock
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock
@ -780,7 +780,7 @@ dependencies = [
 [[package]]
 name = "collab"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -800,7 +800,7 @@ dependencies = [
 [[package]]
 name = "collab-database"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -827,7 +827,7 @@ dependencies = [
 [[package]]
 name = "collab-document"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "collab",
@ -844,7 +844,7 @@ dependencies = [
 [[package]]
 name = "collab-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "bytes",
@ -858,7 +858,7 @@ dependencies = [
 [[package]]
 name = "collab-folder"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "chrono",
@ -893,7 +893,7 @@ dependencies = [
 [[package]]
 name = "collab-plugins"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -923,7 +923,7 @@ dependencies = [
 [[package]]
 name = "collab-user"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "collab",
--- a/frontend/appflowy_tauri/src-tauri/Cargo.toml
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml
@ -67,13 +67,13 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "215
 # To switch to the local path, run:
 # scripts/tool/update_collab_source.sh
 # ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }



--- a/frontend/rust-lib/Cargo.lock
+++ b/frontend/rust-lib/Cargo.lock
@ -686,7 +686,7 @@ dependencies = [
 [[package]]
 name = "collab"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -706,7 +706,7 @@ dependencies = [
 [[package]]
 name = "collab-database"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -733,7 +733,7 @@ dependencies = [
 [[package]]
 name = "collab-document"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "collab",
@ -750,7 +750,7 @@ dependencies = [
 [[package]]
 name = "collab-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "bytes",
@ -764,7 +764,7 @@ dependencies = [
 [[package]]
 name = "collab-folder"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "chrono",
@ -799,7 +799,7 @@ dependencies = [
 [[package]]
 name = "collab-plugins"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "async-trait",
@ -829,7 +829,7 @@ dependencies = [
 [[package]]
 name = "collab-user"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5c425595182ef5a43913484a775fae6f59fe9e52#5c425595182ef5a43913484a775fae6f59fe9e52"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b566a65b3eb3d0abff7c09371cad309341865cde#b566a65b3eb3d0abff7c09371cad309341865cde"
 dependencies = [
 "anyhow",
 "collab",
--- a/frontend/rust-lib/Cargo.toml
+++ b/frontend/rust-lib/Cargo.toml
@ -117,10 +117,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "215
 # To switch to the local path, run:
 # scripts/tool/update_collab_source.sh
 # ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "5c425595182ef5a43913484a775fae6f59fe9e52" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b566a65b3eb3d0abff7c09371cad309341865cde" }
--- a/frontend/rust-lib/collab-integrate/src/collab_builder.rs
+++ b/frontend/rust-lib/collab-integrate/src/collab_builder.rs
@ -194,7 +194,7 @@ impl AppFlowyCollabBuilder {
        object_type,
        collab_db,
        collab_doc_state,
-        &persistence_config,
+        persistence_config,
        build_config,
      )
      .await
@ -222,7 +222,7 @@ impl AppFlowyCollabBuilder {
    object_type: CollabType,
    collab_db: Weak<CollabKVDB>,
    collab_doc_state: CollabDocState,
-    persistence_config: &CollabPersistenceConfig,
+    persistence_config: CollabPersistenceConfig,
    build_config: CollabBuilderConfig,
  ) -> Result<Arc<MutexCollab>, Error> {
    let collab = Arc::new(
@ -232,7 +232,6 @@ impl AppFlowyCollabBuilder {
          uid,
          collab_db.clone(),
          persistence_config.clone(),
-          self.rocksdb_backup.lock().clone(),
        ))
        .with_device_id(self.device_id.clone())
        .build()?,
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs
@ -1,8 +1,9 @@
-use std::sync::Weak;
-
-use diesel::SqliteConnection;
-
+use collab_entity::CollabType;
 use collab_integrate::{CollabSnapshot, PersistenceError, SnapshotPersistence};
+use diesel::dsl::count_star;
+use diesel::internal::derives::multiconnection::chrono;
+use diesel::internal::derives::multiconnection::chrono::Local;
+use diesel::SqliteConnection;
 use flowy_error::FlowyError;
 use flowy_sqlite::{
  insert_or_ignore_into,
@ -11,6 +12,7 @@ use flowy_sqlite::{
 };
 use flowy_user::manager::UserManager;
 use lib_infra::util::timestamp;
+use std::sync::Weak;

 pub struct SnapshotDBImpl(pub Weak<UserManager>);

@ -33,9 +35,10 @@ impl SnapshotPersistence for SnapshotDBImpl {
    &self,
    uid: i64,
    object_id: &str,
-    title: String,
-    snapshot_data: Vec<u8>,
+    collab_type: &CollabType,
+    encoded_v1: Vec<u8>,
  ) -> Result<(), PersistenceError> {
+    let collab_type = collab_type.clone();
    let object_id = object_id.to_string();
    let weak_user_session = self.0.clone();
    tokio::task::spawn_blocking(move || {
@ -49,19 +52,10 @@ impl SnapshotPersistence for SnapshotDBImpl {

        // Save the snapshot data to disk
        let result = CollabSnapshotTableSql::create(
-          CollabSnapshotRow {
-            id: uuid::Uuid::new_v4().to_string(),
-            object_id: object_id.clone(),
-            title,
-            desc: "".to_string(),
-            collab_type: "".to_string(),
-            timestamp: timestamp(),
-            data: snapshot_data,
-          },
+          CollabSnapshotRow::new(object_id.clone(), collab_type.to_string(), encoded_v1),
          &mut conn,
        )
        .map_err(|e| PersistenceError::Internal(e.into()));
-
        if let Err(e) = result {
          tracing::warn!("create snapshot error: {:?}", e);
        }
@ -84,6 +78,20 @@ struct CollabSnapshotRow {
  data: Vec<u8>,
 }

+impl CollabSnapshotRow {
+  pub fn new(object_id: String, collab_type: String, data: Vec<u8>) -> Self {
+    Self {
+      id: uuid::Uuid::new_v4().to_string(),
+      object_id,
+      title: "".to_string(),
+      desc: "".to_string(),
+      collab_type,
+      timestamp: timestamp(),
+      data,
+    }
+  }
+}
+
 impl From<CollabSnapshotRow> for CollabSnapshot {
  fn from(table: CollabSnapshotRow) -> Self {
    Self {
@ -96,7 +104,6 @@ impl From<CollabSnapshotRow> for CollabSnapshot {
 struct CollabSnapshotTableSql;
 impl CollabSnapshotTableSql {
  fn create(row: CollabSnapshotRow, conn: &mut SqliteConnection) -> Result<(), FlowyError> {
-    // Batch insert: https://diesel.rs/guides/all-about-inserts.html
    let values = (
      dsl::id.eq(row.id),
      dsl::object_id.eq(row.object_id),
@ -109,6 +116,16 @@ impl CollabSnapshotTableSql {
    let _ = insert_or_ignore_into(dsl::collab_snapshot)
      .values(values)
      .execute(conn)?;
+
+    // Count the total number of rows
+    // If there are more than 10 rows, delete snapshots older than 5 days
+    let total_rows: i64 = dsl::collab_snapshot.select(count_star()).first(conn)?;
+    if total_rows > 10 {
+      let five_days_ago = Local::now() - chrono::Duration::days(5);
+      let _ =
+        diesel::delete(dsl::collab_snapshot.filter(dsl::timestamp.lt(five_days_ago.timestamp())))
+          .execute(conn)?;
+    };
    Ok(())
  }

--- a/frontend/rust-lib/flowy-database2/src/manager.rs
+++ b/frontend/rust-lib/flowy-database2/src/manager.rs
@ -133,7 +133,7 @@ impl DatabaseManager {
      CollabType::WorkspaceDatabase,
      collab_db.clone(),
      collab_raw_data,
-      &config,
+      config.clone(),
    );
    let workspace_database =
      WorkspaceDatabase::open(uid, collab, collab_db, config, collab_builder);
@ -461,7 +461,7 @@ impl DatabaseCollabService for UserDatabaseCollabServiceImpl {
    object_type: CollabType,
    collab_db: Weak<CollabKVDB>,
    collab_raw_data: CollabDocState,
-    config: &CollabPersistenceConfig,
+    persistence_config: CollabPersistenceConfig,
  ) -> Arc<MutexCollab> {
    block_on(self.collab_builder.build_with_config(
      uid,
@ -469,7 +469,7 @@ impl DatabaseCollabService for UserDatabaseCollabServiceImpl {
      object_type,
      collab_db,
      collab_raw_data,
-      config,
+      persistence_config,
      CollabBuilderConfig::default().sync_enable(true),
    ))
    .unwrap()
--- a/frontend/rust-lib/flowy-document/src/entities.rs
+++ b/frontend/rust-lib/flowy-document/src/entities.rs
@ -398,9 +398,6 @@ pub struct DocumentSnapshotPB {

  #[pb(index = 3)]
  pub created_at: i64,
-
-  #[pb(index = 4)]
-  pub data: Vec<u8>,
 }

 #[derive(Debug, Default, ProtoBuf)]
--- a/frontend/rust-lib/flowy-document/src/manager.rs
+++ b/frontend/rust-lib/flowy-document/src/manager.rs
@ -15,7 +15,7 @@ use parking_lot::Mutex;
 use tracing::{event, instrument};

 use collab_integrate::collab_builder::{AppFlowyCollabBuilder, CollabBuilderConfig};
-use collab_integrate::{CollabKVAction, CollabKVDB};
+use collab_integrate::{CollabKVAction, CollabKVDB, CollabPersistenceConfig};
 use flowy_document_deps::cloud::DocumentCloudService;
 use flowy_error::{internal_error, ErrorCode, FlowyError, FlowyResult};
 use flowy_storage::FileStorageService;
@ -219,7 +219,6 @@ impl DocumentManager {
        snapshot_id: snapshot.snapshot_id,
        snapshot_desc: "".to_string(),
        created_at: snapshot.created_at,
-        data: snapshot.data,
      })
      .collect::<Vec<_>>();

@ -236,12 +235,13 @@ impl DocumentManager {
    let db = self.user.collab_db(uid)?;
    let collab = self
      .collab_builder
-      .build(
+      .build_with_config(
        uid,
        doc_id,
        CollabType::Document,
-        doc_state,
        db,
+        doc_state,
+        CollabPersistenceConfig::default().snapshot_per_update(100),
        CollabBuilderConfig::default().sync_enable(sync_enable),
      )
      .await?;
--- a/frontend/rust-lib/flowy-folder/src/manager.rs
+++ b/frontend/rust-lib/flowy-folder/src/manager.rs
@ -147,7 +147,9 @@ impl FolderManager {
        CollabType::Folder,
        collab_db,
        collab_doc_state,
-        &CollabPersistenceConfig::new().enable_snapshot(true),
+        CollabPersistenceConfig::new()
+          .enable_snapshot(true)
+          .snapshot_per_update(20),
        CollabBuilderConfig::default().sync_enable(true),
      )
      .await?;
--- a/frontend/rust-lib/lib-infra/Cargo.toml
+++ b/frontend/rust-lib/lib-infra/Cargo.toml
@ -21,4 +21,4 @@ tempfile = "3.8.1"
 validator = "0.16.0"

 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-zip = "0.6.6"
+zip = { version = "0.6.6", features = ["deflate"] }
--- a/frontend/rust-lib/lib-infra/src/file_util.rs
+++ b/frontend/rust-lib/lib-infra/src/file_util.rs
@ -79,7 +79,7 @@ pub fn zip_folder(src_path: impl AsRef<Path>, dest_path: &Path) -> io::Result<()

  let file = File::create(dest_path)?;
  let mut zip = ZipWriter::new(file);
-  let options = FileOptions::default().compression_method(zip::CompressionMethod::Stored);
+  let options = FileOptions::default().compression_method(zip::CompressionMethod::Deflated);

  for entry in WalkDir::new(&src_path) {
    let entry = entry?;