feat: Import appflowy data (#4236)

* refactor: traits

* feat: import data

* chore: track database view

* fix: import

* refactor: collab doc state

* refactor: get collab doc state

* feat: batch create collab object

* fix: test

* ci: run docker compose if the server is not up

* chore: bump collab

* chore: update ci

* chore: update ci

* chore: update ci

* chore: implement ui

* chore: implement ui

* chore: implement ui

frontend/rust-lib/event-integration/tests/user/af_cloud_test/anon_user_test.rs

@@ -111,9 +111,10 @@ async fn migrate_anon_user_data_to_af_cloud_test() {
   assert_eq!(anon_third_level_views.len(), 2);
   assert_eq!(user_third_level_views[0].name, "Grid1".to_string());
   assert_eq!(user_third_level_views[1].name, "Grid2".to_string());
-  drop(cleaner);
 
   // check the trash
   assert_eq!(user_trash.items.len(), 1);
   assert_eq!(user_trash.items[0].name, anon_trash.items[0].name);
+
+  drop(cleaner);
 }

frontend/rust-lib/event-integration/tests/user/af_cloud_test/mod.rs

@@ -1,3 +1,4 @@
 mod anon_user_test;
 mod auth_test;
 mod member_test;
+mod sync_third_party_data_test;

frontend/rust-lib/event-integration/tests/user/af_cloud_test/sync_third_party_data_test.rs

@@ -0,0 +1,304 @@
+use crate::util::unzip_history_user_db;
+use assert_json_diff::assert_json_include;
+use collab_entity::CollabType;
+use event_integration::user_event::user_localhost_af_cloud;
+use event_integration::{document_data_from_document_doc_state, EventIntegrationTest};
+use flowy_core::DEFAULT_NAME;
+use serde_json::{json, Value};
+
+#[tokio::test]
+async fn import_appflowy_data_folder_test() {
+  let import_container_name = "040_local".to_string();
+  let (cleaner, user_db_path) =
+    unzip_history_user_db("./tests/asset", &import_container_name).unwrap();
+  // In the 040_local, the structure is:
+  // workspace:
+  //  view: Document1
+  //    view: Document2
+  //      view: Grid1
+  //      view: Grid2
+  user_localhost_af_cloud().await;
+  let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
+  let _ = test.af_cloud_sign_up().await;
+  // after sign up, the initial workspace is created, so the structure is:
+  // workspace:
+  //   view: Getting Started
+
+  test
+    .import_appflowy_data(
+      user_db_path.to_str().unwrap().to_string(),
+      &import_container_name,
+    )
+    .await;
+  // after import, the structure is:
+  // workspace:
+  //   view: Getting Started
+  //   view: 040_local
+  //     view: Document1
+  //        view: Document2
+  //          view: Grid1
+  //          view: Grid2
+  let views = test.get_all_workspace_views().await;
+  assert_eq!(views.len(), 2);
+  assert_eq!(views[1].name, import_container_name);
+
+  let local_child_views = test.get_views(&views[1].id).await.child_views;
+  assert_eq!(local_child_views.len(), 1);
+  assert_eq!(local_child_views[0].name, "Document1");
+
+  let document1_child_views = test.get_views(&local_child_views[0].id).await.child_views;
+  assert_eq!(document1_child_views.len(), 1);
+  assert_eq!(document1_child_views[0].name, "Document2");
+
+  let document2_child_views = test
+    .get_views(&document1_child_views[0].id)
+    .await
+    .child_views;
+  assert_eq!(document2_child_views.len(), 2);
+  assert_eq!(document2_child_views[0].name, "Grid1");
+  assert_eq!(document2_child_views[1].name, "Grid2");
+
+  drop(cleaner);
+}
+
+#[tokio::test]
+async fn import_appflowy_data_folder_test2() {
+  let import_container_name = "040_local_2".to_string();
+  let (cleaner, user_db_path) =
+    unzip_history_user_db("./tests/asset", &import_container_name).unwrap();
+  user_localhost_af_cloud().await;
+  let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
+  let _ = test.af_cloud_sign_up().await;
+  test
+    .import_appflowy_data(
+      user_db_path.to_str().unwrap().to_string(),
+      &import_container_name,
+    )
+    .await;
+
+  let views = test.get_all_workspace_views().await;
+  assert_eq!(views.len(), 2);
+  assert_eq!(views[1].name, import_container_name);
+  assert_040_local_2_import_content(&test, &views[1].id).await;
+
+  drop(cleaner);
+}
+
+#[tokio::test]
+async fn import_appflowy_data_folder_multiple_times_test() {
+  let import_container_name = "040_local_2".to_string();
+  let (cleaner, user_db_path) =
+    unzip_history_user_db("./tests/asset", &import_container_name).unwrap();
+  // In the 040_local_2, the structure is:
+  //  Getting Started
+  //     Doc1
+  //     Doc2
+  //     Grid1
+  //     Doc3
+  //        Doc3_grid_1
+  //        Doc3_grid_2
+  //        Doc3_calendar_1
+  user_localhost_af_cloud().await;
+  let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
+  let _ = test.af_cloud_sign_up().await;
+  test
+    .import_appflowy_data(
+      user_db_path.to_str().unwrap().to_string(),
+      &import_container_name,
+    )
+    .await;
+  // after import, the structure is:
+  //   Getting Started
+  //   040_local_2
+
+  let views = test.get_all_workspace_views().await;
+  assert_eq!(views.len(), 2);
+  assert_eq!(views[1].name, import_container_name);
+  assert_040_local_2_import_content(&test, &views[1].id).await;
+
+  test
+    .import_appflowy_data(
+      user_db_path.to_str().unwrap().to_string(),
+      &import_container_name,
+    )
+    .await;
+  // after import, the structure is:
+  //   Getting Started
+  //   040_local_2
+  //      Getting started
+  //   040_local_2
+  //      Getting started
+  let views = test.get_all_workspace_views().await;
+  assert_eq!(views.len(), 3);
+  assert_eq!(views[2].name, import_container_name);
+  assert_040_local_2_import_content(&test, &views[1].id).await;
+  assert_040_local_2_import_content(&test, &views[2].id).await;
+  drop(cleaner);
+}
+
+async fn assert_040_local_2_import_content(test: &EventIntegrationTest, view_id: &str) {
+  //   040_local_2
+  //      Getting started
+  //         Doc1
+  //         Doc2
+  //         Grid1
+  //         Doc3
+  //            Doc3_grid_1
+  //            Doc3_grid_2
+  //            Doc3_calendar_1
+  let _local_2_child_views = test.get_views(view_id).await.child_views;
+  assert_eq!(_local_2_child_views.len(), 1);
+  assert_eq!(_local_2_child_views[0].name, "Getting started");
+
+  let local_2_getting_started_child_views = test
+    .get_views(&_local_2_child_views[0].id)
+    .await
+    .child_views;
+
+  // Check doc 1 local content
+  let doc_1 = local_2_getting_started_child_views[0].clone();
+  assert_eq!(doc_1.name, "Doc1");
+  let data = test.get_document_data(&doc_1.id).await;
+  assert_json_include!(actual: json!(data), expected: expected_doc_1_json());
+
+  // Check doc 1 remote content
+  let doc_1_doc_state = test
+    .get_collab_doc_state(&doc_1.id, CollabType::Document)
+    .await
+    .unwrap();
+  assert_json_include!(actual:document_data_from_document_doc_state(&doc_1.id, doc_1_doc_state), expected: expected_doc_1_json());
+
+  // Check doc 2 local content
+  let doc_2 = local_2_getting_started_child_views[1].clone();
+  assert_eq!(doc_2.name, "Doc2");
+  let data = test.get_document_data(&doc_2.id).await;
+  assert_json_include!(actual: json!(data), expected: expected_doc_2_json());
+
+  // Check doc 2 remote content
+  let doc_2_doc_state = test.get_document_doc_state(&doc_2.id).await;
+  assert_json_include!(actual:document_data_from_document_doc_state(&doc_2.id, doc_2_doc_state), expected: expected_doc_2_json());
+
+  let grid_1 = local_2_getting_started_child_views[2].clone();
+  assert_eq!(grid_1.name, "Grid1");
+  assert_eq!(
+    test.get_database_export_data(&grid_1.id).await,
+    "Name,Type,Done\n1,A,Yes\n2,,Yes\n3,,No\n"
+  );
+
+  assert_eq!(local_2_getting_started_child_views[3].name, "Doc3");
+
+  let doc_3_child_views = test
+    .get_views(&local_2_getting_started_child_views[3].id)
+    .await
+    .child_views;
+  assert_eq!(doc_3_child_views.len(), 3);
+  assert_eq!(doc_3_child_views[0].name, "doc3_grid_1");
+
+  let doc3_grid_2 = doc_3_child_views[1].clone();
+  assert_eq!(doc3_grid_2.name, "doc3_grid_2");
+  assert_eq!(
+    test.get_database_export_data(&doc3_grid_2.id).await,
+    "Name,Type,Done\n1,A,Yes\n2,,\n,,\n"
+  );
+  assert_eq!(doc_3_child_views[2].name, "doc3_calendar_1");
+}
+
+fn expected_doc_1_json() -> Value {
+  json!({
+    "blocks": {
+      "Rnslggtr6s": {
+        "children": "CoT14jXwTV",
+        "data": {
+          "delta": [
+            {
+              "insert": "Hello Document 1"
+            }
+          ]
+        },
+        "external_id": "hUDq6PrdP1",
+        "external_type": "text",
+        "id": "Rnslggtr6s",
+        "parent": "vxWayiyi2Q",
+        "ty": "paragraph"
+      },
+      "vxWayiyi2Q": {
+        "children": "hAgnEMJtU2",
+        "data": {},
+        "external_id": null,
+        "external_type": null,
+        "id": "vxWayiyi2Q",
+        "parent": "",
+        "ty": "page"
+      }
+    },
+    "meta": {
+      "children_map": {
+        "CoT14jXwTV": [],
+        "hAgnEMJtU2": [
+          "Rnslggtr6s"
+        ]
+      },
+      "text_map": {
+        "hUDq6PrdP1": "[{\"insert\":\"Hello Document 1\"}]",
+        "ujncfD": "[]"
+      }
+    },
+    "page_id": "vxWayiyi2Q"
+  })
+}
+fn expected_doc_2_json() -> Value {
+  json!({
+    "blocks": {
+      "ZVogdaK9yO": {
+        "children": "cc20wCE77N",
+        "data": {},
+        "external_id": null,
+        "external_type": null,
+        "id": "ZVogdaK9yO",
+        "parent": "",
+        "ty": "page"
+      },
+      "bVRuGAvyfp": {
+        "children": "pOVd5xKBal",
+        "data": {
+          "delta": [
+            {
+              "insert": "Hello Document 2"
+            }
+          ]
+        },
+        "external_id": "m7mwLgXzDF",
+        "external_type": "text",
+        "id": "bVRuGAvyfp",
+        "parent": "ZVogdaK9yO",
+        "ty": "paragraph"
+      },
+      "ng2b4I": {
+        "children": "YMaDFs",
+        "data": {
+          "delta": []
+        },
+        "external_id": null,
+        "external_type": null,
+        "id": "ng2b4I",
+        "parent": "ZVogdaK9yO",
+        "ty": "paragraph"
+      }
+    },
+    "meta": {
+      "children_map": {
+        "YMaDFs": [],
+        "cc20wCE77N": [
+          "bVRuGAvyfp",
+          "ng2b4I"
+        ],
+        "pOVd5xKBal": []
+      },
+      "text_map": {
+        "m7mwLgXzDF": "[{\"insert\":\"Hello Document 2\"}]",
+        "qXQmuS": "[]"
+      }
+    },
+    "page_id": "ZVogdaK9yO"
+  })
+}

frontend/rust-lib/event-integration/tests/user/supabase_test/auth_test.rs

@@ -390,7 +390,7 @@ async fn migrate_anon_data_on_cloud_signup() {
       }
 
       assert!(cloud_service
-        .get_collab_update(&database_id, CollabType::Database, &workspace_id)
+        .get_collab_doc_state_db(&database_id, CollabType::Database, &workspace_id)
         .await
         .is_ok());
     }