Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: stuck when importing large csv file (#4878)

Browse Source 
* chore: fix import csv file

* chore: update collab

* chore: bump collab

* chore: workaround for row creating

* fix: test
This commit is contained in:
Nathan.fooo 2024-03-12 10:59:52 +08:00 committed by GitHub
parent c48001bd74
commit 9c3be7e488
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 169 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
frontend/appflowy_tauri/src-tauri/Cargo.lock generated
View File

@ -818,7 +818,7 @@ dependencies = [
[[package]] [[package]]
name = "collab" name = "collab"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
@ -841,7 +841,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-database" name = "collab-database"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
@ -855,6 +855,7 @@ dependencies = [
"lru", "lru",
"nanoid", "nanoid",
"parking_lot 0.12.1", "parking_lot 0.12.1",
"rayon",
"serde", "serde",
"serde_json", "serde_json",
"serde_repr", "serde_repr",
@ -870,7 +871,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-document" name = "collab-document"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"collab", "collab",
@ -889,7 +890,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-entity" name = "collab-entity"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytes", "bytes",
@ -904,7 +905,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-folder" name = "collab-folder"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"chrono", "chrono",
@ -941,7 +942,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-plugins" name = "collab-plugins"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -980,7 +981,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-user" name = "collab-user"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"collab", "collab",
@ -1203,7 +1204,7 @@ dependencies = [
"cssparser-macros", "cssparser-macros",
"dtoa-short", "dtoa-short",
"itoa 1.0.6", "itoa 1.0.6",
"phf 0.8.0", "phf 0.11.2",
"smallvec", "smallvec",
] ]
@ -4781,9 +4782,9 @@ checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9"
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.8.1" version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
dependencies = [ dependencies = [
"either", "either",
"rayon-core", "rayon-core",

14
frontend/appflowy_tauri/src-tauri/Cargo.toml
View File

@ -92,10 +92,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run: # To switch to the local path, run:
# scripts/tool/update_collab_source.sh # scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️ # ⚠️⚠️⚠️️
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }

14
frontend/appflowy_web/wasm-libs/Cargo.toml
View File

@ -65,10 +65,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run: # To switch to the local path, run:
# scripts/tool/update_collab_source.sh # scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️ # ⚠️⚠️⚠️️
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }

19
frontend/rust-lib/Cargo.lock generated
View File

@ -746,7 +746,7 @@ dependencies = [
[[package]] [[package]]
name = "collab" name = "collab"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
@ -769,7 +769,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-database" name = "collab-database"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
@ -783,6 +783,7 @@ dependencies = [
"lru", "lru",
"nanoid", "nanoid",
"parking_lot 0.12.1", "parking_lot 0.12.1",
"rayon",
"serde", "serde",
"serde_json", "serde_json",
"serde_repr", "serde_repr",
@ -798,7 +799,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-document" name = "collab-document"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"collab", "collab",
@ -817,7 +818,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-entity" name = "collab-entity"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytes", "bytes",
@ -832,7 +833,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-folder" name = "collab-folder"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"chrono", "chrono",
@ -869,7 +870,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-plugins" name = "collab-plugins"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -908,7 +909,7 @@ dependencies = [
[[package]] [[package]]
name = "collab-user" name = "collab-user"
version = "0.1.0" version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380" source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"collab", "collab",
@ -4290,9 +4291,9 @@ dependencies = [
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.8.1" version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
dependencies = [ dependencies = [
"either", "either",
"rayon-core", "rayon-core",

14
frontend/rust-lib/Cargo.toml
View File

@ -115,10 +115,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run: # To switch to the local path, run:
# scripts/tool/update_collab_source.sh # scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️ # ⚠️⚠️⚠️️
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" } collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }

24
frontend/rust-lib/event-integration/src/folder_event.rs
View File

@ -8,6 +8,7 @@ use flowy_user::entities::{
}; };
use flowy_user::errors::FlowyError; use flowy_user::errors::FlowyError;
use flowy_user::event_map::UserEvent; use flowy_user::event_map::UserEvent;
use tokio::time::sleep;
use crate::event_builder::EventBuilder; use crate::event_builder::EventBuilder;
use crate::EventIntegrationTest; use crate::EventIntegrationTest;
@ -134,6 +135,15 @@ impl EventIntegrationTest {
.await .await
.parse::<ViewPB>() .parse::<ViewPB>()
} }
pub async fn import_data(&self, data: ImportPB) -> ViewPB {
EventBuilder::new(self.clone())
.event(FolderEvent::ImportData)
.payload(data)
.async_send()
.await
.parse::<ViewPB>()
}
} }
pub struct ViewTest { pub struct ViewTest {
@ -172,15 +182,23 @@ impl ViewTest {
} }
pub async fn new_grid_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self { pub async fn new_grid_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
Self::new(sdk, ViewLayoutPB::Grid, data).await // TODO(nathan): remove this sleep
// workaround for the rows that are created asynchronously
let this = Self::new(sdk, ViewLayoutPB::Grid, data).await;
sleep(tokio::time::Duration::from_secs(2)).await;
this
} }
pub async fn new_board_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self { pub async fn new_board_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
Self::new(sdk, ViewLayoutPB::Board, data).await let this = Self::new(sdk, ViewLayoutPB::Board, data).await;
sleep(tokio::time::Duration::from_secs(2)).await;
this
} }
pub async fn new_calendar_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self { pub async fn new_calendar_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
Self::new(sdk, ViewLayoutPB::Calendar, data).await let this = Self::new(sdk, ViewLayoutPB::Calendar, data).await;
sleep(tokio::time::Duration::from_secs(2)).await;
this
} }
} }

BIN
frontend/rust-lib/event-integration/tests/asset/csv_10240r_15c.csv.zip Normal file
View File

Binary file not shown.

BIN
frontend/rust-lib/event-integration/tests/asset/csv_492r_17c.csv.zip Normal file
View File

Binary file not shown.

55
frontend/rust-lib/event-integration/tests/folder/local_test/import_test.rs Normal file
View File

@ -0,0 +1,55 @@
use crate::util::unzip_history_user_db;
use event_integration::EventIntegrationTest;
use flowy_core::DEFAULT_NAME;
use flowy_folder::entities::{ImportPB, ImportTypePB, ViewLayoutPB};
#[tokio::test]
async fn import_492_row_csv_file_test() {
// csv_500r_15c.csv is a file with 492 rows and 17 columns
let file_name = "csv_492r_17c.csv".to_string();
let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
test.sign_up_as_guest().await;
let workspace_id = test.get_current_workspace().await.id;
let import_data = gen_import_data(file_name, csv_string, workspace_id);
let view = test.import_data(import_data).await;
let database = test.get_database(&view.id).await;
assert_eq!(database.rows.len(), 492);
drop(cleaner);
}
#[tokio::test]
async fn import_10240_row_csv_file_test() {
// csv_22577r_15c.csv is a file with 10240 rows and 15 columns
let file_name = "csv_10240r_15c.csv".to_string();
let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
test.sign_up_as_guest().await;
let workspace_id = test.get_current_workspace().await.id;
let import_data = gen_import_data(file_name, csv_string, workspace_id);
let view = test.import_data(import_data).await;
let database = test.get_database(&view.id).await;
assert_eq!(database.rows.len(), 10240);
drop(cleaner);
}
fn gen_import_data(file_name: String, csv_string: String, workspace_id: String) -> ImportPB {
let import_data = ImportPB {
parent_view_id: workspace_id.clone(),
name: file_name,
data: Some(csv_string.as_bytes().to_vec()),
file_path: None,
view_layout: ViewLayoutPB::Grid,
import_type: ImportTypePB::CSV,
};
import_data
}

1
frontend/rust-lib/event-integration/tests/folder/local_test/mod.rs
View File

@ -1,4 +1,5 @@
mod folder_test; mod folder_test;
mod import_test;
mod script; mod script;
mod subscription_test; mod subscription_test;
mod test; mod test;

7
frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs
View File

@ -360,8 +360,11 @@ impl FolderOperationHandler for DatabaseFolderOperation {
_ => CSVFormat::Original, _ => CSVFormat::Original,
}; };
FutureResult::new(async move { FutureResult::new(async move {
let content = let content = tokio::task::spawn_blocking(move || {
String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))?; String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))
})
.await??;
database_manager database_manager
.import_csv(view_id, content, format) .import_csv(view_id, content, format)
.await?; .await?;

2
frontend/rust-lib/flowy-database2/Cargo.toml
View File

@ -40,7 +40,7 @@ futures.workspace = true
dashmap = "5" dashmap = "5"
anyhow.workspace = true anyhow.workspace = true
async-stream = "0.3.4" async-stream = "0.3.4"
rayon = "1.6.1" rayon = "1.9.0"
nanoid = "0.4.0" nanoid = "0.4.0"
async-trait.workspace = true async-trait.workspace = true
chrono-tz = "0.8.2" chrono-tz = "0.8.2"

6
frontend/rust-lib/flowy-database2/src/entities/database_entities.rs
View File

@ -1,6 +1,6 @@
use collab::core::collab_state::SyncState; use collab::core::collab_state::SyncState;
use collab_database::rows::RowId; use collab_database::rows::RowId;
use collab_database::user::DatabaseViewTracker; use collab_database::user::DatabaseMeta;
use collab_database::views::DatabaseLayout; use collab_database::views::DatabaseLayout;
use flowy_derive::ProtoBuf; use flowy_derive::ProtoBuf;
@ -208,8 +208,8 @@ pub struct DatabaseDescriptionPB {
pub database_id: String, pub database_id: String,
} }
impl From<DatabaseViewTracker> for DatabaseDescriptionPB { impl From<DatabaseMeta> for DatabaseDescriptionPB {
fn from(data: DatabaseViewTracker) -> Self { fn from(data: DatabaseMeta) -> Self {
Self { Self {
database_id: data.database_id, database_id: data.database_id,
} }

2
frontend/rust-lib/flowy-database2/src/manager.rs
View File

@ -168,7 +168,7 @@ impl DatabaseManager {
let mut items = vec![]; let mut items = vec![];
if let Ok(wdb) = self.get_workspace_database().await { if let Ok(wdb) = self.get_workspace_database().await {
items = wdb items = wdb
.get_all_databases() .get_all_database_meta()
.into_iter() .into_iter()
.map(DatabaseDescriptionPB::from) .map(DatabaseDescriptionPB::from)
.collect(); .collect();

3
frontend/rust-lib/flowy-database2/tests/database/share_test/export_test.rs
View File

@ -124,6 +124,9 @@ async fn history_database_import_test() {
"{""data"":""AE"",""field_type"":0}","{""data"":""5"",""field_type"":1}","{""data"":""1671938394"",""field_type"":2}","{""data"":""wQpG"",""field_type"":3}","{""data"":"""",""field_type"":4}","{""data"":""Yes"",""field_type"":5}","{""data"":"""",""field_type"":6}","{""data"":"""",""field_type"":7}" "{""data"":""AE"",""field_type"":0}","{""data"":""5"",""field_type"":1}","{""data"":""1671938394"",""field_type"":2}","{""data"":""wQpG"",""field_type"":3}","{""data"":"""",""field_type"":4}","{""data"":""Yes"",""field_type"":5}","{""data"":"""",""field_type"":6}","{""data"":"""",""field_type"":7}"
"#; "#;
let result = test.import(csv.to_string(), format).await; let result = test.import(csv.to_string(), format).await;
// TODO(nathan): remove this sleep
// workaround for the rows that are created asynchronously
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
let database = test.get_database(&result.database_id).await.unwrap(); let database = test.get_database(&result.database_id).await.unwrap();
let fields = database.get_fields(&result.view_id, None); let fields = database.get_fields(&result.view_id, None);

12
frontend/rust-lib/flowy-folder/src/entities/view.rs
View File

@ -59,14 +59,14 @@ pub struct ViewPB {
pub is_favorite: bool, pub is_favorite: bool,
} }
pub fn view_pb_without_child_views(view: Arc<View>) -> ViewPB { pub fn view_pb_without_child_views(view: View) -> ViewPB {
ViewPB { ViewPB {
id: view.id.clone(), id: view.id,
parent_view_id: view.parent_view_id.clone(), parent_view_id: view.parent_view_id,
name: view.name.clone(), name: view.name,
create_time: view.created_at, create_time: view.created_at,
child_views: Default::default(), child_views: Default::default(),
layout: view.layout.clone().into(), layout: view.layout.into(),
icon: view.icon.clone().map(|icon| icon.into()), icon: view.icon.clone().map(|icon| icon.into()),
is_favorite: view.is_favorite, is_favorite: view.is_favorite,
} }
@ -81,7 +81,7 @@ pub fn view_pb_with_child_views(view: Arc<View>, child_views: Vec<Arc<View>>) ->
create_time: view.created_at, create_time: view.created_at,
child_views: child_views child_views: child_views
.into_iter() .into_iter()
.map(view_pb_without_child_views) .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect(), .collect(),
layout: view.layout.clone().into(), layout: view.layout.clone().into(),
icon: view.icon.clone().map(|icon| icon.into()), icon: view.icon.clone().map(|icon| icon.into()),

13
frontend/rust-lib/flowy-folder/src/event_handler.rs
View File

@ -28,7 +28,7 @@ pub(crate) async fn create_workspace_handler(
.get_views_belong_to(&workspace.id) .get_views_belong_to(&workspace.id)
.await? .await?
.into_iter() .into_iter()
.map(view_pb_without_child_views) .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect::<Vec<ViewPB>>(); .collect::<Vec<ViewPB>>();
data_result_ok(WorkspacePB { data_result_ok(WorkspacePB {
id: workspace.id, id: workspace.id,
@ -85,7 +85,7 @@ pub(crate) async fn create_view_handler(
if set_as_current { if set_as_current {
let _ = folder.set_current_view(&view.id).await; let _ = folder.set_current_view(&view.id).await;
} }
data_result_ok(view_pb_without_child_views(Arc::new(view))) data_result_ok(view_pb_without_child_views(view))
} }
pub(crate) async fn create_orphan_view_handler( pub(crate) async fn create_orphan_view_handler(
@ -99,7 +99,7 @@ pub(crate) async fn create_orphan_view_handler(
if set_as_current { if set_as_current {
let _ = folder.set_current_view(&view.id).await; let _ = folder.set_current_view(&view.id).await;
} }
data_result_ok(view_pb_without_child_views(Arc::new(view))) data_result_ok(view_pb_without_child_views(view))
} }
#[tracing::instrument(level = "debug", skip(data, folder), err)] #[tracing::instrument(level = "debug", skip(data, folder), err)]
@ -313,11 +313,12 @@ pub(crate) async fn delete_all_trash_handler(
pub(crate) async fn import_data_handler( pub(crate) async fn import_data_handler(
data: AFPluginData<ImportPB>, data: AFPluginData<ImportPB>,
folder: AFPluginState<Weak<FolderManager>>, folder: AFPluginState<Weak<FolderManager>>,
) -> Result<(), FlowyError> { ) -> DataResult<ViewPB, FlowyError> {
let folder = upgrade_folder(folder)?; let folder = upgrade_folder(folder)?;
let params: ImportParams = data.into_inner().try_into()?; let params: ImportParams = data.into_inner().try_into()?;
folder.import(params).await?; let view = folder.import(params).await?;
Ok(()) let view_pb = view_pb_without_child_views(view);
data_result_ok(view_pb)
} }
#[tracing::instrument(level = "debug", skip(folder), err)] #[tracing::instrument(level = "debug", skip(folder), err)]

2
frontend/rust-lib/flowy-folder/src/event_map.rs
View File

@ -124,7 +124,7 @@ pub enum FolderEvent {
#[event()] #[event()]
PermanentlyDeleteAllTrashItem = 27, PermanentlyDeleteAllTrashItem = 27,
#[event(input = "ImportPB")] #[event(input = "ImportPB", output = "ViewPB")]
ImportData = 30, ImportData = 30,
#[event(input = "WorkspaceIdPB", output = "RepeatedFolderSnapshotPB")] #[event(input = "WorkspaceIdPB", output = "RepeatedFolderSnapshotPB")]

6
frontend/rust-lib/flowy-folder/src/manager.rs
View File

@ -374,7 +374,7 @@ impl FolderManager {
.views .views
.get_views_belong_to(&workspace.id) .get_views_belong_to(&workspace.id)
.into_iter() .into_iter()
.map(view_pb_without_child_views) .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect::<Vec<ViewPB>>(); .collect::<Vec<ViewPB>>();
WorkspacePB { WorkspacePB {
@ -556,7 +556,7 @@ impl FolderManager {
.send(); .send();
notify_child_views_changed( notify_child_views_changed(
view_pb_without_child_views(view), view_pb_without_child_views(view.as_ref().clone()),
ChildViewChangeReason::Delete, ChildViewChangeReason::Delete,
); );
} }
@ -573,7 +573,7 @@ impl FolderManager {
let favorite_descendant_views: Vec<ViewPB> = all_descendant_views let favorite_descendant_views: Vec<ViewPB> = all_descendant_views
.iter() .iter()
.filter(|view| view.is_favorite) .filter(|view| view.is_favorite)
.map(|view| view_pb_without_child_views(view.clone())) .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect(); .collect();
if !favorite_descendant_views.is_empty() { if !favorite_descendant_views.is_empty() {

11
frontend/rust-lib/flowy-folder/src/manager_observer.rs
View File

@ -32,7 +32,7 @@ pub(crate) fn subscribe_folder_view_changed(
match value { match value {
ViewChange::DidCreateView { view } => { ViewChange::DidCreateView { view } => {
notify_child_views_changed( notify_child_views_changed(
view_pb_without_child_views(Arc::new(view.clone())), view_pb_without_child_views(view.clone()),
ChildViewChangeReason::Create, ChildViewChangeReason::Create,
); );
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id]); notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id]);
@ -40,7 +40,7 @@ pub(crate) fn subscribe_folder_view_changed(
ViewChange::DidDeleteView { views } => { ViewChange::DidDeleteView { views } => {
for view in views { for view in views {
notify_child_views_changed( notify_child_views_changed(
view_pb_without_child_views(view), view_pb_without_child_views(view.as_ref().clone()),
ChildViewChangeReason::Delete, ChildViewChangeReason::Delete,
); );
} }
@ -48,7 +48,7 @@ pub(crate) fn subscribe_folder_view_changed(
ViewChange::DidUpdate { view } => { ViewChange::DidUpdate { view } => {
notify_view_did_change(view.clone()); notify_view_did_change(view.clone());
notify_child_views_changed( notify_child_views_changed(
view_pb_without_child_views(Arc::new(view.clone())), view_pb_without_child_views(view.clone()),
ChildViewChangeReason::Update, ChildViewChangeReason::Update,
); );
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id.clone()]); notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id.clone()]);
@ -190,8 +190,9 @@ pub(crate) fn notify_did_update_workspace(workspace_id: &str, folder: &Folder) {
} }
fn notify_view_did_change(view: View) -> Option<()> { fn notify_view_did_change(view: View) -> Option<()> {
let view_pb = view_pb_without_child_views(Arc::new(view.clone())); let view_id = view.id.clone();
send_notification(&view.id, FolderNotification::DidUpdateView) let view_pb = view_pb_without_child_views(view);
send_notification(&view_id, FolderNotification::DidUpdateView)
.payload(view_pb) .payload(view_pb)
.send(); .send();
None None

8
frontend/rust-lib/flowy-user/src/anon_user/migrate_anon_user_collab.rs
View File

@ -10,7 +10,7 @@ use collab_database::database::{
is_database_collab, mut_database_views_with_collab, reset_inline_view_id, is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
}; };
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId}; use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
use collab_database::user::DatabaseViewTrackerList; use collab_database::user::DatabaseMetaList;
use collab_folder::{Folder, UserId}; use collab_folder::{Folder, UserId};
use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::kv::KVTransactionDB;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
@ -163,9 +163,9 @@ where
let new_uid = new_user_session.user_id; let new_uid = new_user_session.user_id;
let new_object_id = &new_user_session.user_workspace.workspace_database_object_id; let new_object_id = &new_user_session.user_workspace.workspace_database_object_id;
let array = DatabaseViewTrackerList::from_collab(&database_with_views_collab); let array = DatabaseMetaList::from_collab(&database_with_views_collab);
for database_view_tracker in array.get_all_database_tracker() { for database_metas in array.get_all_database_meta() {
array.update_database(&database_view_tracker.database_id, |update| { array.update_database(&database_metas.database_id, |update| {
let new_linked_views = update let new_linked_views = update
.linked_views .linked_views
.iter() .iter()

10
frontend/rust-lib/flowy-user/src/anon_user/sync_supabase_user_collab.rs
View File

@ -8,7 +8,7 @@ use collab::core::collab::MutexCollab;
use collab::preclude::Collab; use collab::preclude::Collab;
use collab_database::database::get_database_row_ids; use collab_database::database::get_database_row_ids;
use collab_database::rows::database_row_document_id_from_row_id; use collab_database::rows::database_row_document_id_from_row_id;
use collab_database::user::{get_all_database_view_trackers, DatabaseViewTracker}; use collab_database::user::{get_all_database_meta, DatabaseMeta};
use collab_entity::{CollabObject, CollabType}; use collab_entity::{CollabObject, CollabType};
use collab_folder::{Folder, View, ViewLayout}; use collab_folder::{Folder, View, ViewLayout};
use collab_plugins::local_storage::kv::KVTransactionDB; use collab_plugins::local_storage::kv::KVTransactionDB;
@ -75,7 +75,7 @@ pub async fn sync_supabase_user_data_to_cloud(
fn sync_view( fn sync_view(
uid: i64, uid: i64,
folder: Arc<MutexFolder>, folder: Arc<MutexFolder>,
database_records: Vec<Arc<DatabaseViewTracker>>, database_records: Vec<Arc<DatabaseMeta>>,
workspace_id: String, workspace_id: String,
device_id: String, device_id: String,
view: Arc<View>, view: Arc<View>,
@ -297,7 +297,7 @@ async fn sync_database_views(
database_views_aggregate_id: &str, database_views_aggregate_id: &str,
collab_db: &Arc<CollabKVDB>, collab_db: &Arc<CollabKVDB>,
user_service: Arc<dyn UserCloudService>, user_service: Arc<dyn UserCloudService>,
) -> Vec<Arc<DatabaseViewTracker>> { ) -> Vec<Arc<DatabaseMeta>> {
let collab_object = CollabObject::new( let collab_object = CollabObject::new(
uid, uid,
database_views_aggregate_id.to_string(), database_views_aggregate_id.to_string(),
@ -317,7 +317,7 @@ async fn sync_database_views(
}) })
.map(|_| { .map(|_| {
( (
get_all_database_view_trackers(&collab), get_all_database_meta(&collab),
collab.encode_collab_v1().doc_state, collab.encode_collab_v1().doc_state,
) )
}) })
@ -357,7 +357,7 @@ fn collab_type_from_view_layout(view_layout: &ViewLayout) -> CollabType {
fn object_id_from_view( fn object_id_from_view(
view: &Arc<View>, view: &Arc<View>,
database_records: &[Arc<DatabaseViewTracker>], database_records: &[Arc<DatabaseMeta>],
) -> Result<String, Error> { ) -> Result<String, Error> {
if view.layout.is_database() { if view.layout.is_database() {
match database_records match database_records

10
frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs
View File

@ -14,7 +14,7 @@ use collab_database::database::{
is_database_collab, mut_database_views_with_collab, reset_inline_view_id, is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
}; };
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId}; use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
use collab_database::user::DatabaseViewTrackerList; use collab_database::user::DatabaseMetaList;
use collab_document::document_data::default_document_collab_data; use collab_document::document_data::default_document_collab_data;
use collab_entity::CollabType; use collab_entity::CollabType;
use collab_folder::{Folder, UserId, View, ViewIdentifier, ViewLayout}; use collab_folder::{Folder, UserId, View, ViewIdentifier, ViewLayout};
@ -280,11 +280,11 @@ where
) )
})?; })?;
let array = DatabaseViewTrackerList::from_collab(&database_view_tracker_collab); let array = DatabaseMetaList::from_collab(&database_view_tracker_collab);
for database_view_tracker in array.get_all_database_tracker() { for database_metas in array.get_all_database_meta() {
database_view_ids_by_database_id.insert( database_view_ids_by_database_id.insert(
old_to_new_id_map.renew_id(&database_view_tracker.database_id), old_to_new_id_map.renew_id(&database_metas.database_id),
database_view_tracker database_metas
.linked_views .linked_views
.into_iter() .into_iter()
.map(|view_id| old_to_new_id_map.renew_id(&view_id)) .map(|view_id| old_to_new_id_map.renew_id(&view_id))