mirror of
https://github.com/AppFlowy-IO/AppFlowy.git
synced 2024-08-30 18:12:39 +00:00
fix: stuck when importing large csv file (#4878)
* chore: fix import csv file * chore: update collab * chore: bump collab * chore: workaround for row creating * fix: test
This commit is contained in:
Nathan.fooo
GitHub
parent
c48001bd74
commit
9c3be7e488
21
frontend/appflowy_tauri/src-tauri/Cargo.lock
generated
21
frontend/appflowy_tauri/src-tauri/Cargo.lock
generated
@ -818,7 +818,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@ -841,7 +841,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-database"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@ -855,6 +855,7 @@ dependencies = [
|
||||
"lru",
|
||||
"nanoid",
|
||||
"parking_lot 0.12.1",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_repr",
|
||||
@ -870,7 +871,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-document"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"collab",
|
||||
@ -889,7 +890,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-entity"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
@ -904,7 +905,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-folder"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
@ -941,7 +942,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-plugins"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@ -980,7 +981,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-user"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"collab",
|
||||
@ -1203,7 +1204,7 @@ dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa 1.0.6",
|
||||
"phf 0.8.0",
|
||||
"phf 0.11.2",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
@ -4781,9 +4782,9 @@ checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051"
|
||||
checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
|
||||
@ -92,10 +92,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
|
||||
# To switch to the local path, run:
|
||||
# scripts/tool/update_collab_source.sh
|
||||
# ⚠️⚠️⚠️️
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
|
||||
@ -65,10 +65,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
|
||||
# To switch to the local path, run:
|
||||
# scripts/tool/update_collab_source.sh
|
||||
# ⚠️⚠️⚠️️
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
|
||||
19
frontend/rust-lib/Cargo.lock
generated
19
frontend/rust-lib/Cargo.lock
generated
@ -746,7 +746,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@ -769,7 +769,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-database"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
@ -783,6 +783,7 @@ dependencies = [
|
||||
"lru",
|
||||
"nanoid",
|
||||
"parking_lot 0.12.1",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_repr",
|
||||
@ -798,7 +799,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-document"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"collab",
|
||||
@ -817,7 +818,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-entity"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
@ -832,7 +833,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-folder"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
@ -869,7 +870,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-plugins"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@ -908,7 +909,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "collab-user"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"collab",
|
||||
@ -4290,9 +4291,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051"
|
||||
checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
|
||||
@ -115,10 +115,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
|
||||
# To switch to the local path, run:
|
||||
# scripts/tool/update_collab_source.sh
|
||||
# ⚠️⚠️⚠️️
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
|
||||
|
||||
@ -8,6 +8,7 @@ use flowy_user::entities::{
|
||||
};
|
||||
use flowy_user::errors::FlowyError;
|
||||
use flowy_user::event_map::UserEvent;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::event_builder::EventBuilder;
|
||||
use crate::EventIntegrationTest;
|
||||
@ -134,6 +135,15 @@ impl EventIntegrationTest {
|
||||
.await
|
||||
.parse::<ViewPB>()
|
||||
}
|
||||
|
||||
pub async fn import_data(&self, data: ImportPB) -> ViewPB {
|
||||
EventBuilder::new(self.clone())
|
||||
.event(FolderEvent::ImportData)
|
||||
.payload(data)
|
||||
.async_send()
|
||||
.await
|
||||
.parse::<ViewPB>()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ViewTest {
|
||||
@ -172,15 +182,23 @@ impl ViewTest {
|
||||
}
|
||||
|
||||
pub async fn new_grid_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
|
||||
Self::new(sdk, ViewLayoutPB::Grid, data).await
|
||||
// TODO(nathan): remove this sleep
|
||||
// workaround for the rows that are created asynchronously
|
||||
let this = Self::new(sdk, ViewLayoutPB::Grid, data).await;
|
||||
sleep(tokio::time::Duration::from_secs(2)).await;
|
||||
this
|
||||
}
|
||||
|
||||
pub async fn new_board_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
|
||||
Self::new(sdk, ViewLayoutPB::Board, data).await
|
||||
let this = Self::new(sdk, ViewLayoutPB::Board, data).await;
|
||||
sleep(tokio::time::Duration::from_secs(2)).await;
|
||||
this
|
||||
}
|
||||
|
||||
pub async fn new_calendar_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
|
||||
Self::new(sdk, ViewLayoutPB::Calendar, data).await
|
||||
let this = Self::new(sdk, ViewLayoutPB::Calendar, data).await;
|
||||
sleep(tokio::time::Duration::from_secs(2)).await;
|
||||
this
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@ -0,0 +1,55 @@
|
||||
use crate::util::unzip_history_user_db;
|
||||
use event_integration::EventIntegrationTest;
|
||||
use flowy_core::DEFAULT_NAME;
|
||||
use flowy_folder::entities::{ImportPB, ImportTypePB, ViewLayoutPB};
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_492_row_csv_file_test() {
|
||||
// csv_500r_15c.csv is a file with 492 rows and 17 columns
|
||||
let file_name = "csv_492r_17c.csv".to_string();
|
||||
let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
|
||||
|
||||
let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
|
||||
let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
|
||||
test.sign_up_as_guest().await;
|
||||
|
||||
let workspace_id = test.get_current_workspace().await.id;
|
||||
let import_data = gen_import_data(file_name, csv_string, workspace_id);
|
||||
|
||||
let view = test.import_data(import_data).await;
|
||||
let database = test.get_database(&view.id).await;
|
||||
assert_eq!(database.rows.len(), 492);
|
||||
drop(cleaner);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn import_10240_row_csv_file_test() {
|
||||
// csv_22577r_15c.csv is a file with 10240 rows and 15 columns
|
||||
let file_name = "csv_10240r_15c.csv".to_string();
|
||||
let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
|
||||
|
||||
let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
|
||||
let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
|
||||
test.sign_up_as_guest().await;
|
||||
|
||||
let workspace_id = test.get_current_workspace().await.id;
|
||||
let import_data = gen_import_data(file_name, csv_string, workspace_id);
|
||||
|
||||
let view = test.import_data(import_data).await;
|
||||
let database = test.get_database(&view.id).await;
|
||||
assert_eq!(database.rows.len(), 10240);
|
||||
|
||||
drop(cleaner);
|
||||
}
|
||||
|
||||
fn gen_import_data(file_name: String, csv_string: String, workspace_id: String) -> ImportPB {
|
||||
let import_data = ImportPB {
|
||||
parent_view_id: workspace_id.clone(),
|
||||
name: file_name,
|
||||
data: Some(csv_string.as_bytes().to_vec()),
|
||||
file_path: None,
|
||||
view_layout: ViewLayoutPB::Grid,
|
||||
import_type: ImportTypePB::CSV,
|
||||
};
|
||||
import_data
|
||||
}
|
||||
@ -1,4 +1,5 @@
|
||||
mod folder_test;
|
||||
mod import_test;
|
||||
mod script;
|
||||
mod subscription_test;
|
||||
mod test;
|
||||
|
||||
@ -360,8 +360,11 @@ impl FolderOperationHandler for DatabaseFolderOperation {
|
||||
_ => CSVFormat::Original,
|
||||
};
|
||||
FutureResult::new(async move {
|
||||
let content =
|
||||
String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))?;
|
||||
let content = tokio::task::spawn_blocking(move || {
|
||||
String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))
|
||||
})
|
||||
.await??;
|
||||
|
||||
database_manager
|
||||
.import_csv(view_id, content, format)
|
||||
.await?;
|
||||
|
||||
@ -40,7 +40,7 @@ futures.workspace = true
|
||||
dashmap = "5"
|
||||
anyhow.workspace = true
|
||||
async-stream = "0.3.4"
|
||||
rayon = "1.6.1"
|
||||
rayon = "1.9.0"
|
||||
nanoid = "0.4.0"
|
||||
async-trait.workspace = true
|
||||
chrono-tz = "0.8.2"
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use collab::core::collab_state::SyncState;
|
||||
use collab_database::rows::RowId;
|
||||
use collab_database::user::DatabaseViewTracker;
|
||||
use collab_database::user::DatabaseMeta;
|
||||
use collab_database::views::DatabaseLayout;
|
||||
|
||||
use flowy_derive::ProtoBuf;
|
||||
@ -208,8 +208,8 @@ pub struct DatabaseDescriptionPB {
|
||||
pub database_id: String,
|
||||
}
|
||||
|
||||
impl From<DatabaseViewTracker> for DatabaseDescriptionPB {
|
||||
fn from(data: DatabaseViewTracker) -> Self {
|
||||
impl From<DatabaseMeta> for DatabaseDescriptionPB {
|
||||
fn from(data: DatabaseMeta) -> Self {
|
||||
Self {
|
||||
database_id: data.database_id,
|
||||
}
|
||||
|
||||
@ -168,7 +168,7 @@ impl DatabaseManager {
|
||||
let mut items = vec![];
|
||||
if let Ok(wdb) = self.get_workspace_database().await {
|
||||
items = wdb
|
||||
.get_all_databases()
|
||||
.get_all_database_meta()
|
||||
.into_iter()
|
||||
.map(DatabaseDescriptionPB::from)
|
||||
.collect();
|
||||
|
||||
@ -124,6 +124,9 @@ async fn history_database_import_test() {
|
||||
"{""data"":""AE"",""field_type"":0}","{""data"":""5"",""field_type"":1}","{""data"":""1671938394"",""field_type"":2}","{""data"":""wQpG"",""field_type"":3}","{""data"":"""",""field_type"":4}","{""data"":""Yes"",""field_type"":5}","{""data"":"""",""field_type"":6}","{""data"":"""",""field_type"":7}"
|
||||
"#;
|
||||
let result = test.import(csv.to_string(), format).await;
|
||||
// TODO(nathan): remove this sleep
|
||||
// workaround for the rows that are created asynchronously
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
|
||||
let database = test.get_database(&result.database_id).await.unwrap();
|
||||
|
||||
let fields = database.get_fields(&result.view_id, None);
|
||||
|
||||
@ -59,14 +59,14 @@ pub struct ViewPB {
|
||||
pub is_favorite: bool,
|
||||
}
|
||||
|
||||
pub fn view_pb_without_child_views(view: Arc<View>) -> ViewPB {
|
||||
pub fn view_pb_without_child_views(view: View) -> ViewPB {
|
||||
ViewPB {
|
||||
id: view.id.clone(),
|
||||
parent_view_id: view.parent_view_id.clone(),
|
||||
name: view.name.clone(),
|
||||
id: view.id,
|
||||
parent_view_id: view.parent_view_id,
|
||||
name: view.name,
|
||||
create_time: view.created_at,
|
||||
child_views: Default::default(),
|
||||
layout: view.layout.clone().into(),
|
||||
layout: view.layout.into(),
|
||||
icon: view.icon.clone().map(|icon| icon.into()),
|
||||
is_favorite: view.is_favorite,
|
||||
}
|
||||
@ -81,7 +81,7 @@ pub fn view_pb_with_child_views(view: Arc<View>, child_views: Vec<Arc<View>>) ->
|
||||
create_time: view.created_at,
|
||||
child_views: child_views
|
||||
.into_iter()
|
||||
.map(view_pb_without_child_views)
|
||||
.map(|view| view_pb_without_child_views(view.as_ref().clone()))
|
||||
.collect(),
|
||||
layout: view.layout.clone().into(),
|
||||
icon: view.icon.clone().map(|icon| icon.into()),
|
||||
|
||||
@ -28,7 +28,7 @@ pub(crate) async fn create_workspace_handler(
|
||||
.get_views_belong_to(&workspace.id)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(view_pb_without_child_views)
|
||||
.map(|view| view_pb_without_child_views(view.as_ref().clone()))
|
||||
.collect::<Vec<ViewPB>>();
|
||||
data_result_ok(WorkspacePB {
|
||||
id: workspace.id,
|
||||
@ -85,7 +85,7 @@ pub(crate) async fn create_view_handler(
|
||||
if set_as_current {
|
||||
let _ = folder.set_current_view(&view.id).await;
|
||||
}
|
||||
data_result_ok(view_pb_without_child_views(Arc::new(view)))
|
||||
data_result_ok(view_pb_without_child_views(view))
|
||||
}
|
||||
|
||||
pub(crate) async fn create_orphan_view_handler(
|
||||
@ -99,7 +99,7 @@ pub(crate) async fn create_orphan_view_handler(
|
||||
if set_as_current {
|
||||
let _ = folder.set_current_view(&view.id).await;
|
||||
}
|
||||
data_result_ok(view_pb_without_child_views(Arc::new(view)))
|
||||
data_result_ok(view_pb_without_child_views(view))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(data, folder), err)]
|
||||
@ -313,11 +313,12 @@ pub(crate) async fn delete_all_trash_handler(
|
||||
pub(crate) async fn import_data_handler(
|
||||
data: AFPluginData<ImportPB>,
|
||||
folder: AFPluginState<Weak<FolderManager>>,
|
||||
) -> Result<(), FlowyError> {
|
||||
) -> DataResult<ViewPB, FlowyError> {
|
||||
let folder = upgrade_folder(folder)?;
|
||||
let params: ImportParams = data.into_inner().try_into()?;
|
||||
folder.import(params).await?;
|
||||
Ok(())
|
||||
let view = folder.import(params).await?;
|
||||
let view_pb = view_pb_without_child_views(view);
|
||||
data_result_ok(view_pb)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(folder), err)]
|
||||
|
||||
@ -124,7 +124,7 @@ pub enum FolderEvent {
|
||||
#[event()]
|
||||
PermanentlyDeleteAllTrashItem = 27,
|
||||
|
||||
#[event(input = "ImportPB")]
|
||||
#[event(input = "ImportPB", output = "ViewPB")]
|
||||
ImportData = 30,
|
||||
|
||||
#[event(input = "WorkspaceIdPB", output = "RepeatedFolderSnapshotPB")]
|
||||
|
||||
@ -374,7 +374,7 @@ impl FolderManager {
|
||||
.views
|
||||
.get_views_belong_to(&workspace.id)
|
||||
.into_iter()
|
||||
.map(view_pb_without_child_views)
|
||||
.map(|view| view_pb_without_child_views(view.as_ref().clone()))
|
||||
.collect::<Vec<ViewPB>>();
|
||||
|
||||
WorkspacePB {
|
||||
@ -556,7 +556,7 @@ impl FolderManager {
|
||||
.send();
|
||||
|
||||
notify_child_views_changed(
|
||||
view_pb_without_child_views(view),
|
||||
view_pb_without_child_views(view.as_ref().clone()),
|
||||
ChildViewChangeReason::Delete,
|
||||
);
|
||||
}
|
||||
@ -573,7 +573,7 @@ impl FolderManager {
|
||||
let favorite_descendant_views: Vec<ViewPB> = all_descendant_views
|
||||
.iter()
|
||||
.filter(|view| view.is_favorite)
|
||||
.map(|view| view_pb_without_child_views(view.clone()))
|
||||
.map(|view| view_pb_without_child_views(view.as_ref().clone()))
|
||||
.collect();
|
||||
|
||||
if !favorite_descendant_views.is_empty() {
|
||||
|
||||
@ -32,7 +32,7 @@ pub(crate) fn subscribe_folder_view_changed(
|
||||
match value {
|
||||
ViewChange::DidCreateView { view } => {
|
||||
notify_child_views_changed(
|
||||
view_pb_without_child_views(Arc::new(view.clone())),
|
||||
view_pb_without_child_views(view.clone()),
|
||||
ChildViewChangeReason::Create,
|
||||
);
|
||||
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id]);
|
||||
@ -40,7 +40,7 @@ pub(crate) fn subscribe_folder_view_changed(
|
||||
ViewChange::DidDeleteView { views } => {
|
||||
for view in views {
|
||||
notify_child_views_changed(
|
||||
view_pb_without_child_views(view),
|
||||
view_pb_without_child_views(view.as_ref().clone()),
|
||||
ChildViewChangeReason::Delete,
|
||||
);
|
||||
}
|
||||
@ -48,7 +48,7 @@ pub(crate) fn subscribe_folder_view_changed(
|
||||
ViewChange::DidUpdate { view } => {
|
||||
notify_view_did_change(view.clone());
|
||||
notify_child_views_changed(
|
||||
view_pb_without_child_views(Arc::new(view.clone())),
|
||||
view_pb_without_child_views(view.clone()),
|
||||
ChildViewChangeReason::Update,
|
||||
);
|
||||
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id.clone()]);
|
||||
@ -190,8 +190,9 @@ pub(crate) fn notify_did_update_workspace(workspace_id: &str, folder: &Folder) {
|
||||
}
|
||||
|
||||
fn notify_view_did_change(view: View) -> Option<()> {
|
||||
let view_pb = view_pb_without_child_views(Arc::new(view.clone()));
|
||||
send_notification(&view.id, FolderNotification::DidUpdateView)
|
||||
let view_id = view.id.clone();
|
||||
let view_pb = view_pb_without_child_views(view);
|
||||
send_notification(&view_id, FolderNotification::DidUpdateView)
|
||||
.payload(view_pb)
|
||||
.send();
|
||||
None
|
||||
|
||||
@ -10,7 +10,7 @@ use collab_database::database::{
|
||||
is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
|
||||
};
|
||||
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
|
||||
use collab_database::user::DatabaseViewTrackerList;
|
||||
use collab_database::user::DatabaseMetaList;
|
||||
use collab_folder::{Folder, UserId};
|
||||
use collab_plugins::local_storage::kv::KVTransactionDB;
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
@ -163,9 +163,9 @@ where
|
||||
let new_uid = new_user_session.user_id;
|
||||
let new_object_id = &new_user_session.user_workspace.workspace_database_object_id;
|
||||
|
||||
let array = DatabaseViewTrackerList::from_collab(&database_with_views_collab);
|
||||
for database_view_tracker in array.get_all_database_tracker() {
|
||||
array.update_database(&database_view_tracker.database_id, |update| {
|
||||
let array = DatabaseMetaList::from_collab(&database_with_views_collab);
|
||||
for database_metas in array.get_all_database_meta() {
|
||||
array.update_database(&database_metas.database_id, |update| {
|
||||
let new_linked_views = update
|
||||
.linked_views
|
||||
.iter()
|
||||
|
||||
@ -8,7 +8,7 @@ use collab::core::collab::MutexCollab;
|
||||
use collab::preclude::Collab;
|
||||
use collab_database::database::get_database_row_ids;
|
||||
use collab_database::rows::database_row_document_id_from_row_id;
|
||||
use collab_database::user::{get_all_database_view_trackers, DatabaseViewTracker};
|
||||
use collab_database::user::{get_all_database_meta, DatabaseMeta};
|
||||
use collab_entity::{CollabObject, CollabType};
|
||||
use collab_folder::{Folder, View, ViewLayout};
|
||||
use collab_plugins::local_storage::kv::KVTransactionDB;
|
||||
@ -75,7 +75,7 @@ pub async fn sync_supabase_user_data_to_cloud(
|
||||
fn sync_view(
|
||||
uid: i64,
|
||||
folder: Arc<MutexFolder>,
|
||||
database_records: Vec<Arc<DatabaseViewTracker>>,
|
||||
database_records: Vec<Arc<DatabaseMeta>>,
|
||||
workspace_id: String,
|
||||
device_id: String,
|
||||
view: Arc<View>,
|
||||
@ -297,7 +297,7 @@ async fn sync_database_views(
|
||||
database_views_aggregate_id: &str,
|
||||
collab_db: &Arc<CollabKVDB>,
|
||||
user_service: Arc<dyn UserCloudService>,
|
||||
) -> Vec<Arc<DatabaseViewTracker>> {
|
||||
) -> Vec<Arc<DatabaseMeta>> {
|
||||
let collab_object = CollabObject::new(
|
||||
uid,
|
||||
database_views_aggregate_id.to_string(),
|
||||
@ -317,7 +317,7 @@ async fn sync_database_views(
|
||||
})
|
||||
.map(|_| {
|
||||
(
|
||||
get_all_database_view_trackers(&collab),
|
||||
get_all_database_meta(&collab),
|
||||
collab.encode_collab_v1().doc_state,
|
||||
)
|
||||
})
|
||||
@ -357,7 +357,7 @@ fn collab_type_from_view_layout(view_layout: &ViewLayout) -> CollabType {
|
||||
|
||||
fn object_id_from_view(
|
||||
view: &Arc<View>,
|
||||
database_records: &[Arc<DatabaseViewTracker>],
|
||||
database_records: &[Arc<DatabaseMeta>],
|
||||
) -> Result<String, Error> {
|
||||
if view.layout.is_database() {
|
||||
match database_records
|
||||
|
||||
@ -14,7 +14,7 @@ use collab_database::database::{
|
||||
is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
|
||||
};
|
||||
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
|
||||
use collab_database::user::DatabaseViewTrackerList;
|
||||
use collab_database::user::DatabaseMetaList;
|
||||
use collab_document::document_data::default_document_collab_data;
|
||||
use collab_entity::CollabType;
|
||||
use collab_folder::{Folder, UserId, View, ViewIdentifier, ViewLayout};
|
||||
@ -280,11 +280,11 @@ where
|
||||
)
|
||||
})?;
|
||||
|
||||
let array = DatabaseViewTrackerList::from_collab(&database_view_tracker_collab);
|
||||
for database_view_tracker in array.get_all_database_tracker() {
|
||||
let array = DatabaseMetaList::from_collab(&database_view_tracker_collab);
|
||||
for database_metas in array.get_all_database_meta() {
|
||||
database_view_ids_by_database_id.insert(
|
||||
old_to_new_id_map.renew_id(&database_view_tracker.database_id),
|
||||
database_view_tracker
|
||||
old_to_new_id_map.renew_id(&database_metas.database_id),
|
||||
database_metas
|
||||
.linked_views
|
||||
.into_iter()
|
||||
.map(|view_id| old_to_new_id_map.renew_id(&view_id))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user