chore: versionize migration (#5060)

frontend/appflowy_tauri/src-tauri/Cargo.lock

@@ -1826,6 +1826,7 @@ dependencies = [
  "lib-infra",
  "lib-log",
  "parking_lot 0.12.1",
+ "semver",
  "serde",
  "serde_json",
  "serde_repr",
@@ -2199,6 +2200,7 @@ dependencies = [
  "once_cell",
  "parking_lot 0.12.1",
  "protobuf",
+ "semver",
  "serde",
  "serde_json",
  "serde_repr",

frontend/rust-lib/Cargo.lock

@@ -1784,6 +1784,7 @@ dependencies = [
  "lib-infra",
  "lib-log",
  "parking_lot 0.12.1",
+ "semver",
  "serde",
  "serde_json",
  "serde_repr",
@@ -2173,6 +2174,7 @@ dependencies = [
  "quickcheck_macros",
  "rand 0.8.5",
  "rand_core 0.6.4",
+ "semver",
  "serde",
  "serde_json",
  "serde_repr",

frontend/rust-lib/flowy-core/Cargo.toml

@@ -48,6 +48,7 @@ serde_repr.workspace = true
 futures.workspace = true
 walkdir = "2.4.0"
 sysinfo = "0.30.5"
+semver = "1.0.22"
 
 [features]
 default = ["rev-sqlite"]

frontend/rust-lib/flowy-core/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(unused_doc_comments)]
 
 use flowy_storage::ObjectStorageService;
+use semver::Version;
 use std::sync::Arc;
 use std::time::Duration;
 use sysinfo::System;
@@ -93,6 +94,7 @@ impl AppFlowyCore {
       server_type,
       Arc::downgrade(&store_preference),
     ));
+    let app_version = Version::parse(&config.app_version).unwrap_or_else(|_| Version::new(0, 5, 4));
 
     event!(tracing::Level::DEBUG, "Init managers",);
     let (
@@ -115,6 +117,7 @@ impl AppFlowyCore {
         &config.storage_path,
         &config.application_path,
         &config.device_id,
+        app_version,
       );
 
       let authenticate_user = Arc::new(AuthenticateUser::new(

frontend/rust-lib/flowy-user/Cargo.toml

@@ -46,6 +46,7 @@ uuid.workspace = true
 chrono = { workspace = true,  default-features = false, features = ["clock"] }
 base64 = "^0.21"
 tokio-stream = "0.1.14"
+semver = "1.0.22"
 
 [dev-dependencies]
 nanoid = "0.4.0"

frontend/rust-lib/flowy-user/src/migrations/document_empty_content.rs

@@ -6,6 +6,7 @@ use collab_document::document::Document;
 use collab_document::document_data::default_document_data;
 use collab_folder::{Folder, View};
 use collab_plugins::local_storage::kv::KVTransactionDB;
+use semver::Version;
 use tracing::{event, instrument};
 
 use collab_integrate::{CollabKVAction, CollabKVDB, PersistenceError};
@@ -24,6 +25,10 @@ impl UserDataMigration for HistoricalEmptyDocumentMigration {
     "historical_empty_document"
   }
 
+  fn applies_to_version(&self, _version: &Version) -> bool {
+    true
+  }
+
   #[instrument(name = "HistoricalEmptyDocumentMigration", skip_all, err)]
   fn run(
     &self,

frontend/rust-lib/flowy-user/src/migrations/migration.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use chrono::NaiveDateTime;
 use diesel::{RunQueryDsl, SqliteConnection};
+use semver::Version;
 
 use collab_integrate::CollabKVDB;
 use flowy_error::FlowyResult;
@@ -47,6 +48,7 @@ impl UserLocalDataMigration {
     self,
     migrations: Vec<Box<dyn UserDataMigration>>,
     authenticator: &Authenticator,
+    app_version: Option<Version>,
   ) -> FlowyResult<Vec<String>> {
     let mut applied_migrations = vec![];
     let mut conn = self.sqlite_pool.get()?;
@@ -57,11 +59,17 @@ impl UserLocalDataMigration {
         .iter()
         .any(|record| record.migration_name == migration.name())
       {
+        if let Some(app_version) = app_version.as_ref() {
+          if !migration.applies_to_version(app_version) {
+            continue;
+          }
+        }
+
         let migration_name = migration.name().to_string();
         if !duplicated_names.contains(&migration_name) {
           migration.run(&self.session, &self.collab_db, authenticator)?;
           applied_migrations.push(migration.name().to_string());
-          save_record(&mut conn, &migration_name);
+          save_migration_record(&mut conn, &migration_name);
           duplicated_names.push(migration_name);
         } else {
           tracing::error!("Duplicated migration name: {}", migration_name);
@@ -75,6 +83,9 @@ impl UserLocalDataMigration {
 pub trait UserDataMigration {
   /// Migration with the same name will be skipped
   fn name(&self) -> &str;
+  /// Returns bool value whether the migration should be applied to the current app version
+  /// true if the migration should be applied, false otherwise
+  fn applies_to_version(&self, app_version: &Version) -> bool;
   fn run(
     &self,
     user: &Session,
@@ -83,7 +94,7 @@ pub trait UserDataMigration {
   ) -> FlowyResult<()>;
 }
 
-fn save_record(conn: &mut SqliteConnection, migration_name: &str) {
+pub(crate) fn save_migration_record(conn: &mut SqliteConnection, migration_name: &str) {
   let new_record = NewUserDataMigrationRecord {
     migration_name: migration_name.to_string(),
   };

frontend/rust-lib/flowy-user/src/migrations/workspace_and_favorite_v1.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use collab_folder::Folder;
 use collab_plugins::local_storage::kv::{KVTransactionDB, PersistenceError};
+use semver::Version;
 use tracing::instrument;
 
 use collab_integrate::{CollabKVAction, CollabKVDB};
@@ -22,6 +23,10 @@ impl UserDataMigration for FavoriteV1AndWorkspaceArrayMigration {
     "workspace_favorite_v1_and_workspace_array_migration"
   }
 
+  fn applies_to_version(&self, _app_version: &Version) -> bool {
+    true
+  }
+
   #[instrument(name = "FavoriteV1AndWorkspaceArrayMigration", skip_all, err)]
   fn run(
     &self,

frontend/rust-lib/flowy-user/src/migrations/workspace_trash_v1.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use collab_folder::Folder;
 use collab_plugins::local_storage::kv::{KVTransactionDB, PersistenceError};
+use semver::Version;
 use tracing::instrument;
 
 use collab_integrate::{CollabKVAction, CollabKVDB};
@@ -20,6 +21,10 @@ impl UserDataMigration for WorkspaceTrashMapToSectionMigration {
     "workspace_trash_map_to_section_migration"
   }
 
+  fn applies_to_version(&self, _app_version: &Version) -> bool {
+    true
+  }
+
   #[instrument(name = "WorkspaceTrashMapToSectionMigration", skip_all, err)]
   fn run(
     &self,

frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs

@@ -80,6 +80,7 @@ pub(crate) fn get_appflowy_data_folder_import_context(path: &str) -> anyhow::Res
     &imported_user,
     imported_collab_db.clone(),
     imported_sqlite_db.get_pool(),
+    None,
   );
 
   Ok(ImportContext {

frontend/rust-lib/flowy-user/src/services/entities.rs

@@ -5,6 +5,7 @@ use std::path::PathBuf;
 use crate::services::db::UserDBPath;
 use base64::engine::general_purpose::PAD;
 use base64::engine::GeneralPurpose;
+use semver::Version;
 
 pub const URL_SAFE_ENGINE: GeneralPurpose = GeneralPurpose::new(&URL_SAFE, PAD);
 #[derive(Clone)]
@@ -19,18 +20,26 @@ pub struct UserConfig {
   pub device_id: String,
   /// Used as the key of `Session` when saving session information to KV.
   pub(crate) session_cache_key: String,
+  pub app_version: Version,
 }
 
 impl UserConfig {
   /// The `root_dir` represents as the root of the user folders. It must be unique for each
   /// users.
-  pub fn new(name: &str, storage_path: &str, application_path: &str, device_id: &str) -> Self {
+  pub fn new(
+    name: &str,
+    storage_path: &str,
+    application_path: &str,
+    device_id: &str,
+    app_version: Version,
+  ) -> Self {
     let session_cache_key = format!("{}_session_cache", name);
     Self {
       storage_path: storage_path.to_owned(),
       application_path: application_path.to_owned(),
       session_cache_key,
       device_id: device_id.to_owned(),
+      app_version,
     }
   }
 

frontend/rust-lib/flowy-user/src/user_manager/manager.rs

@@ -11,13 +11,14 @@ use flowy_sqlite::{query_dsl::*, DBConnection, ExpressionMethods};
 use flowy_user_pub::cloud::{UserCloudServiceProvider, UserUpdate};
 use flowy_user_pub::entities::*;
 use flowy_user_pub::workspace_service::UserWorkspaceService;
+use semver::Version;
 use serde_json::Value;
 use std::string::ToString;
 use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
 use std::sync::{Arc, Weak};
 use tokio::sync::{Mutex, RwLock};
 use tokio_stream::StreamExt;
-use tracing::{debug, error, event, info, instrument, warn};
+use tracing::{debug, error, event, info, instrument, trace, warn};
 
 use lib_dispatch::prelude::af_spawn;
 use lib_infra::box_any::BoxAny;
@@ -26,7 +27,9 @@ use crate::anon_user::{migration_anon_user_on_sign_up, sync_supabase_user_data_t
 use crate::entities::{AuthStateChangedPB, AuthStatePB, UserProfilePB, UserSettingPB};
 use crate::event_map::{DefaultUserStatusCallback, UserStatusCallback};
 use crate::migrations::document_empty_content::HistoricalEmptyDocumentMigration;
-use crate::migrations::migration::{UserDataMigration, UserLocalDataMigration};
+use crate::migrations::migration::{
+  save_migration_record, UserDataMigration, UserLocalDataMigration,
+};
 use crate::migrations::workspace_and_favorite_v1::FavoriteV1AndWorkspaceArrayMigration;
 use crate::migrations::workspace_trash_v1::WorkspaceTrashMapToSectionMigration;
 use crate::migrations::AnonUser;
@@ -246,7 +249,13 @@ impl UserManager {
         self.authenticate_user.database.get_pool(session.user_id),
       ) {
         (Ok(collab_db), Ok(sqlite_pool)) => {
-          run_collab_data_migration(&session, &user, collab_db, sqlite_pool);
+          run_collab_data_migration(
+            &session,
+            &user,
+            collab_db,
+            sqlite_pool,
+            Some(self.authenticate_user.user_config.app_version.clone()),
+          );
         },
         _ => error!("Failed to get collab db or sqlite pool"),
       }
@@ -425,6 +434,17 @@ impl UserManager {
       .await?;
 
     if response.is_new_user {
+      // For new user, we don't need to run the migrations
+      if let Ok(pool) = self
+        .authenticate_user
+        .database
+        .get_pool(new_session.user_id)
+      {
+        mark_all_migrations_as_applied(&pool);
+      } else {
+        error!("Failed to get pool for user {}", new_session.user_id);
+      }
+
       if let Some(old_user) = migration_user {
         event!(
           tracing::Level::INFO,
@@ -827,22 +847,39 @@ fn remove_user_token(uid: i64, mut conn: DBConnection) -> FlowyResult<()> {
   Ok(())
 }
 
+fn collab_migration_list() -> Vec<Box<dyn UserDataMigration>> {
+  // ⚠️The order of migrations is crucial. If you're adding a new migration, please ensure
+  // it's appended to the end of the list.
+  vec![
+    Box::new(HistoricalEmptyDocumentMigration),
+    Box::new(FavoriteV1AndWorkspaceArrayMigration),
+    Box::new(WorkspaceTrashMapToSectionMigration),
+  ]
+}
+
+fn mark_all_migrations_as_applied(sqlite_pool: &Arc<ConnectionPool>) {
+  if let Ok(mut conn) = sqlite_pool.get() {
+    for migration in collab_migration_list() {
+      save_migration_record(&mut conn, migration.name());
+    }
+    info!("Mark all migrations as applied");
+  }
+}
+
 pub(crate) fn run_collab_data_migration(
   session: &Session,
   user: &UserProfile,
   collab_db: Arc<CollabKVDB>,
   sqlite_pool: Arc<ConnectionPool>,
+  version: Option<Version>,
+) {
+  trace!("Run collab data migration: {:?}", version);
+  let migrations = collab_migration_list();
+  match UserLocalDataMigration::new(session.clone(), collab_db, sqlite_pool).run(
+    migrations,
+    &user.authenticator,
+    version,
   ) {
-  // ⚠️The order of migrations is crucial. If you're adding a new migration, please ensure
-  // it's appended to the end of the list.
-  let migrations: Vec<Box<dyn UserDataMigration>> = vec![
-    Box::new(HistoricalEmptyDocumentMigration),
-    Box::new(FavoriteV1AndWorkspaceArrayMigration),
-    Box::new(WorkspaceTrashMapToSectionMigration),
-  ];
-  match UserLocalDataMigration::new(session.clone(), collab_db, sqlite_pool)
-    .run(migrations, &user.authenticator)
-  {
     Ok(applied_migrations) => {
       if !applied_migrations.is_empty() {
         info!("Did apply migrations: {:?}", applied_migrations);