Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Get started doc migration (#3102)

Browse Source 
* feat: migrate empty document

* chore: update collab rev

* chore: fmt
This commit is contained in:
Nathan.fooo 2023-08-03 09:14:52 +08:00 committed by GitHub
parent a40c639a96
commit 03b8f2ccb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 546 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
frontend/appflowy_tauri/src-tauri/Cargo.lock generated
View File

@ -105,7 +105,7 @@ checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
[[package]]
name = "appflowy-integrate"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"collab",
@ -1021,7 +1021,7 @@ dependencies = [
[[package]]
name = "collab"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"bytes",
@ -1039,7 +1039,7 @@ dependencies = [
[[package]]
name = "collab-client-ws"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bytes",
"collab-sync",
@ -1057,7 +1057,7 @@ dependencies = [
[[package]]
name = "collab-database"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"async-trait",
@ -1084,7 +1084,7 @@ dependencies = [
[[package]]
name = "collab-derive"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"proc-macro2",
"quote",
@ -1096,7 +1096,7 @@ dependencies = [
[[package]]
name = "collab-document"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"collab",
@ -1115,7 +1115,7 @@ dependencies = [
[[package]]
name = "collab-folder"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"chrono",
@ -1135,7 +1135,7 @@ dependencies = [
[[package]]
name = "collab-persistence"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bincode",
"chrono",
@ -1155,7 +1155,7 @@ dependencies = [
[[package]]
name = "collab-plugins"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"async-trait",
@ -1185,7 +1185,7 @@ dependencies = [
[[package]]
name = "collab-sync"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=12811d#12811d26a96330f6c1acaa8815f1d8d61ca3aa61"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bytes",
"collab",
@ -1507,6 +1507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
dependencies = [
"byteorder",
"chrono",
"diesel_derives",
"libsqlite3-sys",
]
@ -2119,6 +2120,7 @@ dependencies = [
"bytes",
"chrono",
"collab",
"collab-document",
"collab-folder",
"diesel",
"diesel_derives",

13
frontend/appflowy_tauri/src-tauri/Cargo.toml
View File

@ -34,12 +34,13 @@ default = ["custom-protocol"]
custom-protocol = ["tauri/custom-protocol"]
[patch.crates-io]
appflowy-integrate = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-database = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-document = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-folder = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-plugins = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-persistence = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
#collab = { path = "../../AppFlowy-Collab/collab" }
#collab-folder = { path = "../../AppFlowy-Collab/collab-folder" }

3
frontend/rust-lib/.gitignore vendored
View File

@ -15,4 +15,5 @@ bin/
.idea/
AppFlowy-Collab/
.env
.env.**
.env.**
**/unit_test**

137
frontend/rust-lib/Cargo.lock generated
View File

@ -17,6 +17,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aes"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures",
]
[[package]]
name = "ahash"
version = "0.7.6"
@ -85,7 +96,7 @@ checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
[[package]]
name = "appflowy-integrate"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"collab",
@ -576,6 +587,12 @@ dependencies = [
"vsimd",
]
[[package]]
name = "base64ct"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bincode"
version = "1.3.3"
@ -744,6 +761,16 @@ dependencies = [
"either",
]
[[package]]
name = "bzip2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
dependencies = [
"bzip2-sys",
"libc",
]
[[package]]
name = "bzip2-sys"
version = "0.1.11+1.0.8"
@ -839,6 +866,16 @@ dependencies = [
"phf_codegen 0.11.1",
]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common",
"inout",
]
[[package]]
name = "clang-sys"
version = "1.6.1"
@ -888,7 +925,7 @@ dependencies = [
[[package]]
name = "collab"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"bytes",
@ -906,7 +943,7 @@ dependencies = [
[[package]]
name = "collab-client-ws"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bytes",
"collab-sync",
@ -924,7 +961,7 @@ dependencies = [
[[package]]
name = "collab-database"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"async-trait",
@ -951,7 +988,7 @@ dependencies = [
[[package]]
name = "collab-derive"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"proc-macro2",
"quote",
@ -963,7 +1000,7 @@ dependencies = [
[[package]]
name = "collab-document"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"collab",
@ -982,7 +1019,7 @@ dependencies = [
[[package]]
name = "collab-folder"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"chrono",
@ -1002,7 +1039,7 @@ dependencies = [
[[package]]
name = "collab-persistence"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bincode",
"chrono",
@ -1022,7 +1059,7 @@ dependencies = [
[[package]]
name = "collab-plugins"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"anyhow",
"async-trait",
@ -1052,7 +1089,7 @@ dependencies = [
[[package]]
name = "collab-sync"
version = "0.1.0"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f9df5b9#f9df5b9b5bf1e74c305aafcaf57b7b18493bded5"
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=e9a50f#e9a50f780fa97b1e3fce6ffe7720e9585d674974"
dependencies = [
"bytes",
"collab",
@ -1134,6 +1171,12 @@ dependencies = [
"tracing-subscriber 0.3.16",
]
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -1350,6 +1393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
dependencies = [
"byteorder",
"chrono",
"diesel_derives",
"libsqlite3-sys",
]
@ -1949,6 +1993,7 @@ dependencies = [
"tokio-postgres",
"tracing",
"uuid",
"zip",
]
[[package]]
@ -1960,6 +2005,7 @@ dependencies = [
"bytes",
"chrono",
"collab",
"collab-document",
"collab-folder",
"diesel",
"diesel_derives",
@ -2518,6 +2564,15 @@ dependencies = [
"serde",
]
[[package]]
name = "inout"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
dependencies = [
"generic-array",
]
[[package]]
name = "instant"
version = "0.1.12"
@ -3126,6 +3181,29 @@ dependencies = [
"regex",
]
[[package]]
name = "password-hash"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700"
dependencies = [
"base64ct",
"rand_core 0.6.4",
"subtle",
]
[[package]]
name = "pbkdf2"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917"
dependencies = [
"digest",
"hmac",
"password-hash",
"sha2",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@ -5531,6 +5609,45 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"aes",
"byteorder",
"bzip2",
"constant_time_eq",
"crc32fast",
"crossbeam-utils",
"flate2",
"hmac",
"pbkdf2",
"sha1",
"time 0.3.21",
"zstd",
]
[[package]]
name = "zstd"
version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "5.0.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.8+zstd.1.5.5"

13
frontend/rust-lib/Cargo.toml
View File

@ -38,12 +38,12 @@ opt-level = 3
incremental = false
[patch.crates-io]
appflowy-integrate = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-database = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-document = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-folder = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab-plugins = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f9df5b9"}
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
appflowy-integrate = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "e9a50f" }
#collab = { path = "../AppFlowy-Collab/collab" }
#collab-folder = { path = "../AppFlowy-Collab/collab-folder" }
@ -51,3 +51,4 @@ collab-plugins = {git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev =
#collab-document = { path = "../AppFlowy-Collab/collab-document" }
#collab-plugins = { path = "../AppFlowy-Collab/collab-plugins" }
#appflowy-integrate = { path = "../AppFlowy-Collab/appflowy-integrate" }

54
frontend/rust-lib/flowy-document2/src/document_data.rs
View File

@ -1,13 +1,7 @@
use std::{collections::HashMap, vec};
use collab_document::blocks::{Block, DocumentData, DocumentMeta};
use nanoid::nanoid;
use crate::entities::{BlockPB, ChildrenPB, DocumentDataPB, MetaPB};
pub const PAGE: &str = "page";
pub const PARAGRAPH_BLOCK_TYPE: &str = "paragraph";
impl From<DocumentData> for DocumentDataPB {
fn from(data: DocumentData) -> Self {
let blocks = data
@ -58,54 +52,6 @@ impl From<DocumentDataPB> for DocumentData {
}
}
/// The default document data.
/// The default document data is a document with a page block and a text block.
pub fn default_document_data() -> DocumentData {
let page_type = PAGE.to_string();
let text_type = PARAGRAPH_BLOCK_TYPE.to_string();
let mut blocks: HashMap<String, Block> = HashMap::new();
let mut meta: HashMap<String, Vec<String>> = HashMap::new();
// page block
let page_id = nanoid!(10);
let children_id = nanoid!(10);
let root = Block {
id: page_id.clone(),
ty: page_type,
parent: "".to_string(),
children: children_id.clone(),
external_id: None,
external_type: None,
data: HashMap::new(),
};
blocks.insert(page_id.clone(), root);
// text block
let text_block_id = nanoid!(10);
let text_block_children_id = nanoid!(10);
let text_block = Block {
id: text_block_id.clone(),
ty: text_type,
parent: page_id.clone(),
children: text_block_children_id.clone(),
external_id: None,
external_type: None,
data: HashMap::new(),
};
blocks.insert(text_block_id.clone(), text_block);
// meta
meta.insert(children_id, vec![text_block_id]);
meta.insert(text_block_children_id, vec![]);
DocumentData {
page_id,
blocks,
meta: DocumentMeta { children_map: meta },
}
}
impl From<Block> for BlockPB {
fn from(block: Block) -> Self {
Self {

3
frontend/rust-lib/flowy-document2/src/manager.rs
View File

@ -6,14 +6,15 @@ use appflowy_integrate::{CollabType, RocksCollabDB};
use collab::core::collab::MutexCollab;
use collab_document::blocks::DocumentData;
use collab_document::document::Document;
use collab_document::document_data::default_document_data;
use collab_document::YrsDocAction;
use parking_lot::RwLock;
use crate::document::MutexDocument;
use flowy_document_deps::cloud::DocumentCloudService;
use flowy_error::{internal_error, FlowyError, FlowyResult};
use crate::entities::DocumentSnapshotPB;
use crate::{document::MutexDocument, document_data::default_document_data};
pub trait DocumentUser: Send + Sync {
fn user_id(&self) -> Result<i64, FlowyError>;

3
frontend/rust-lib/flowy-document2/tests/document/document_insert_test.rs
View File

@ -1,8 +1,7 @@
use std::{collections::HashMap, vec};
use collab_document::blocks::{Block, BlockAction, BlockActionPayload, BlockActionType};
use flowy_document2::document_data::PARAGRAPH_BLOCK_TYPE;
use collab_document::document_data::PARAGRAPH_BLOCK_TYPE;
use crate::document::util;
use crate::document::util::gen_id;

3
frontend/rust-lib/flowy-document2/tests/document/document_redo_undo_test.rs
View File

@ -1,8 +1,7 @@
use std::collections::HashMap;
use collab_document::blocks::{Block, BlockAction, BlockActionPayload, BlockActionType};
use flowy_document2::document_data::{default_document_data, PARAGRAPH_BLOCK_TYPE};
use collab_document::document_data::{default_document_data, PARAGRAPH_BLOCK_TYPE};
use crate::document::util::{gen_document_id, gen_id, DocumentTest};

3
frontend/rust-lib/flowy-document2/tests/document/document_test.rs
View File

@ -1,10 +1,9 @@
use std::{collections::HashMap, vec};
use collab_document::blocks::{Block, BlockAction, BlockActionPayload, BlockActionType};
use collab_document::document_data::{default_document_data, PARAGRAPH_BLOCK_TYPE};
use serde_json::{json, to_value, Value};
use flowy_document2::document_data::{default_document_data, PARAGRAPH_BLOCK_TYPE};
use crate::document::util::{gen_document_id, gen_id, DocumentTest};
#[tokio::test]

2
frontend/rust-lib/flowy-document2/tests/document/util.rs
View File

@ -5,13 +5,13 @@ use std::sync::Arc;
use appflowy_integrate::collab_builder::{AppFlowyCollabBuilder, DefaultCollabStorageProvider};
use appflowy_integrate::RocksCollabDB;
use collab_document::blocks::DocumentData;
use collab_document::document_data::default_document_data;
use nanoid::nanoid;
use parking_lot::Once;
use tempfile::TempDir;
use tracing_subscriber::{fmt::Subscriber, util::SubscriberInitExt, EnvFilter};
use flowy_document2::document::MutexDocument;
use flowy_document2::document_data::default_document_data;
use flowy_document2::manager::{DocumentManager, DocumentUser};
use flowy_document_deps::cloud::*;

2
frontend/rust-lib/flowy-sqlite/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
diesel = { version = "1.4.8", features = ["sqlite"] }
diesel = { version = "1.4.8", features = ["sqlite", "chrono"] }
diesel_derives = { version = "1.4.1", features = ["sqlite"] }
diesel_migrations = { version = "1.4.0", features = ["sqlite"] }
tracing = { version = "0.1", features = ["log"] }

2
frontend/rust-lib/flowy-sqlite/migrations/2023-08-02-083250_user_migration/down.sql Normal file
View File

@ -0,0 +1,2 @@
-- This file should undo anything in `up.sql`
DROP TABLE user_data_migration_records;

6
frontend/rust-lib/flowy-sqlite/migrations/2023-08-02-083250_user_migration/up.sql Normal file
View File

@ -0,0 +1,6 @@
-- Your SQL goes here
CREATE TABLE user_data_migration_records (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
migration_name TEXT NOT NULL,
executed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);

15
frontend/rust-lib/flowy-sqlite/src/schema.rs
View File

@ -12,6 +12,14 @@ diesel::table! {
}
}
diesel::table! {
user_data_migration_records (id) {
id -> Integer,
migration_name -> Text,
executed_at -> Timestamp,
}
}
diesel::table! {
user_table (id) {
id -> Text,
@ -35,4 +43,9 @@ diesel::table! {
}
}
diesel::allow_tables_to_appear_in_same_query!(collab_snapshot, user_table, user_workspace_table,);
diesel::allow_tables_to_appear_in_same_query!(
collab_snapshot,
user_data_migration_records,
user_table,
user_workspace_table,
);

1
frontend/rust-lib/flowy-test/Cargo.toml
View File

@ -43,6 +43,7 @@ collab-database = { version = "0.1.0" }
collab-plugins = { version = "0.1.0" }
assert-json-diff = "2.0.2"
tokio-postgres = { version = "0.7.8" }
zip = "0.6.6"
[features]
default = ["cloud_test"]

35
frontend/rust-lib/flowy-test/src/lib.rs
View File

@ -10,6 +10,7 @@ use parking_lot::RwLock;
use protobuf::ProtobufError;
use tokio::sync::broadcast::{channel, Sender};
use crate::document::document_event::OpenDocumentData;
use flowy_core::{AppFlowyCore, AppFlowyCoreConfig};
use flowy_database2::entities::*;
use flowy_database2::event_map::DatabaseEvent;
@ -43,7 +44,17 @@ pub struct FlowyCoreTest {
impl Default for FlowyCoreTest {
fn default() -> Self {
let temp_dir = temp_dir();
let config = AppFlowyCoreConfig::new(temp_dir.to_str().unwrap(), nanoid!(6)).log_filter(
Self::new_with_user_data_path(temp_dir.to_str().unwrap(), nanoid!(6))
}
}
impl FlowyCoreTest {
pub fn new() -> Self {
Self::default()
}
pub fn new_with_user_data_path(path: &str, name: String) -> Self {
let config = AppFlowyCoreConfig::new(path, name).log_filter(
"info",
vec!["flowy_test".to_string(), "lib_dispatch".to_string()],
);
@ -51,10 +62,9 @@ impl Default for FlowyCoreTest {
let inner = std::thread::spawn(|| AppFlowyCore::new(config))
.join()
.unwrap();
let auth_type = Arc::new(RwLock::new(AuthTypePB::Local));
let notification_sender = TestNotificationSender::new();
let auth_type = Arc::new(RwLock::new(AuthTypePB::Local));
register_notification_sender(notification_sender.clone());
std::mem::forget(inner.dispatcher());
Self {
inner,
@ -63,12 +73,6 @@ impl Default for FlowyCoreTest {
cleaner: Arc::new(RwLock::new(None)),
}
}
}
impl FlowyCoreTest {
pub fn new() -> Self {
Self::default()
}
pub async fn new_with_guest_user() -> Self {
let test = Self::default();
@ -268,6 +272,19 @@ impl FlowyCoreTest {
.await;
}
pub async fn open_document(&self, doc_id: String) -> OpenDocumentData {
let payload = OpenDocumentPayloadPB {
document_id: doc_id.clone(),
};
let data = EventBuilder::new(self.clone())
.event(DocumentEvent::OpenDocument)
.payload(payload)
.async_send()
.await
.parse::<DocumentDataPB>();
OpenDocumentData { id: doc_id, data }
}
pub async fn create_board(&self, parent_id: &str, name: String, initial_data: Vec<u8>) -> ViewPB {
let payload = CreateViewPayloadPB {
parent_view_id: parent_id.to_string(),

23
frontend/rust-lib/flowy-test/tests/user/migration_test/document_test.rs Normal file
View File

@ -0,0 +1,23 @@
use crate::user::migration_test::util::unzip_history_user_db;
use flowy_core::DEFAULT_NAME;
use flowy_folder2::entities::ViewLayoutPB;
use flowy_test::FlowyCoreTest;
#[tokio::test]
async fn migrate_historical_empty_document_test() {
let (cleaner, user_db_path) = unzip_history_user_db("historical_empty_document").unwrap();
let test = FlowyCoreTest::new_with_user_data_path(
user_db_path.to_str().unwrap(),
DEFAULT_NAME.to_string(),
);
let views = test.get_all_workspace_views().await;
assert_eq!(views.len(), 3);
for view in views {
assert_eq!(view.layout, ViewLayoutPB::Document);
let doc = test.open_document(view.id).await;
println!("doc: {:?}", doc.data);
}
drop(cleaner);
}

BIN
frontend/rust-lib/flowy-test/tests/user/migration_test/history_user_db/historical_empty_document.zip Normal file
View File

Binary file not shown.

2
frontend/rust-lib/flowy-test/tests/user/migration_test/mod.rs Normal file
View File

@ -0,0 +1,2 @@
mod document_test;
mod util;

64
frontend/rust-lib/flowy-test/tests/user/migration_test/util.rs Normal file
View File

@ -0,0 +1,64 @@
use nanoid::nanoid;
use std::fs::{create_dir_all, File};
use std::io::copy;
use std::path::{Path, PathBuf};
use zip::ZipArchive;
pub fn unzip_history_user_db(folder_name: &str) -> std::io::Result<(Cleaner, PathBuf)> {
// Open the zip file
let zip_file_path = format!(
"./tests/user/migration_test/history_user_db/{}.zip",
folder_name
);
let reader = File::open(zip_file_path)?;
let output_folder_path = format!(
"./tests/user/migration_test/history_user_db/unit_test_{}",
nanoid!(6)
);
// Create a ZipArchive from the file
let mut archive = ZipArchive::new(reader)?;
// Iterate through each file in the zip
for i in 0..archive.len() {
let mut file = archive.by_index(i)?;
let output_path = Path::new(&output_folder_path).join(file.mangled_name());
if file.name().ends_with('/') {
// Create directory
create_dir_all(&output_path)?;
} else {
// Write file
if let Some(p) = output_path.parent() {
if !p.exists() {
create_dir_all(p)?;
}
}
let mut outfile = File::create(&output_path)?;
copy(&mut file, &mut outfile)?;
}
}
let path = format!("{}/{}", output_folder_path, folder_name);
Ok((
Cleaner::new(PathBuf::from(output_folder_path)),
PathBuf::from(path),
))
}
pub struct Cleaner(PathBuf);
impl Cleaner {
pub fn new(dir: PathBuf) -> Self {
Cleaner(dir)
}
fn cleanup(dir: &PathBuf) {
let _ = std::fs::remove_dir_all(dir);
}
}
impl Drop for Cleaner {
fn drop(&mut self) {
Self::cleanup(&self.0)
}
}

1
frontend/rust-lib/flowy-test/tests/user/mod.rs
View File

@ -1,4 +1,5 @@
mod local_test;
mod migration_test;
#[cfg(feature = "cloud_test")]
mod supabase_test;

1
frontend/rust-lib/flowy-user/Cargo.toml
View File

@ -16,6 +16,7 @@ lib-dispatch = { path = "../lib-dispatch" }
appflowy-integrate = { version = "0.1.0" }
collab = { version = "0.1.0" }
collab-folder = { version = "0.1.0" }
collab-document = { version = "0.1.0" }
flowy-user-deps = { path = "../flowy-user-deps" }
tracing = { version = "0.1", features = ["log"] }

1
frontend/rust-lib/flowy-user/src/lib.rs
View File

@ -4,6 +4,7 @@ extern crate flowy_sqlite;
pub mod entities;
mod event_handler;
pub mod event_map;
mod migrations;
mod notification;
pub mod protobuf;
pub mod services;

7
frontend/rust-lib/flowy-user/src/migrations/define.rs Normal file
View File

@ -0,0 +1,7 @@
use crate::services::session_serde::Session;
use flowy_user_deps::entities::UserProfile;
pub struct UserMigrationContext {
pub user_profile: UserProfile,
pub session: Session,
}

51
frontend/rust-lib/flowy-user/src/migrations/historical_document.rs Normal file
View File

@ -0,0 +1,51 @@
use crate::migrations::migration::UserDataMigration;
use crate::services::session_serde::Session;
use appflowy_integrate::{RocksCollabDB, YrsDocAction};
use collab::core::collab::MutexCollab;
use collab::core::origin::{CollabClient, CollabOrigin};
use collab_document::document::Document;
use collab_document::document_data::default_document_data;
use collab_folder::core::Folder;
use flowy_error::{internal_error, FlowyResult};
use std::sync::Arc;
/// Migrate the first level documents of the workspace by inserting documents
pub struct HistoricalEmptyDocumentMigration;
impl UserDataMigration for HistoricalEmptyDocumentMigration {
fn name(&self) -> &str {
"historical_empty_document"
}
fn run(&self, session: &Session, collab_db: &Arc<RocksCollabDB>) -> FlowyResult<()> {
let write_txn = collab_db.write_txn();
if let Ok(updates) = write_txn.get_all_updates(session.user_id, &session.user_workspace.id) {
let origin = CollabOrigin::Client(CollabClient::new(session.user_id, ""));
// Deserialize the folder from the raw data
let folder =
Folder::from_collab_raw_data(origin.clone(), updates, &session.user_workspace.id, vec![])?;
// Migration the first level documents of the workspace
let migration_views = folder.get_workspace_views(&session.user_workspace.id);
for view in migration_views {
// Read all updates of the view
if let Ok(view_updates) = write_txn.get_all_updates(session.user_id, &view.id) {
if let Err(_) = Document::from_updates(origin.clone(), view_updates, &view.id, vec![]) {
// Create a document with default data
let document_data = default_document_data();
let collab = Arc::new(MutexCollab::new(origin.clone(), &view.id, vec![]));
if let Ok(document) = Document::create_with_data(collab.clone(), document_data) {
// Remove all old updates and then insert the new update
let (doc_state, sv) = document.get_collab().encode_as_update_v1();
write_txn
.flush_doc_with(session.user_id, &view.id, &doc_state, &sv)
.map_err(internal_error)?;
}
}
}
}
}
write_txn.commit_transaction().map_err(internal_error)?;
Ok(())
}
}

105
frontend/rust-lib/flowy-user/src/services/user_data_migration.rs → frontend/rust-lib/flowy-user/src/migrations/local_user_to_cloud.rs
View File

@ -6,68 +6,59 @@ use collab::core::origin::{CollabClient, CollabOrigin};
use collab::preclude::Collab;
use collab_folder::core::{Folder, FolderData};
use crate::migrations::UserMigrationContext;
use flowy_error::{ErrorCode, FlowyError, FlowyResult};
use flowy_user_deps::entities::UserProfile;
use crate::services::session_serde::Session;
pub struct UserDataMigration();
pub struct UserMigrationContext {
pub user_profile: UserProfile,
pub session: Session,
}
impl UserDataMigration {
pub fn migration(
old_user: &UserMigrationContext,
old_collab_db: &Arc<RocksCollabDB>,
new_user: &UserMigrationContext,
new_collab_db: &Arc<RocksCollabDB>,
) -> FlowyResult<Option<FolderData>> {
let mut folder_data = None;
new_collab_db
.with_write_txn(|w_txn| {
let read_txn = old_collab_db.read_txn();
if let Ok(object_ids) = read_txn.get_all_docs() {
// Migration of all objects
for object_id in object_ids {
tracing::debug!("migrate object: {:?}", object_id);
if let Ok(updates) = read_txn.get_all_updates(old_user.session.user_id, &object_id) {
// If the object is a folder, migrate the folder data
if object_id == old_user.session.user_workspace.id {
folder_data = migrate_folder(
old_user.session.user_id,
&object_id,
&new_user.session.user_workspace.id,
updates,
);
} else if object_id == old_user.session.user_workspace.database_storage_id {
migrate_database_storage(
old_user.session.user_id,
&object_id,
new_user.session.user_id,
&new_user.session.user_workspace.database_storage_id,
updates,
w_txn,
);
} else {
migrate_object(
old_user.session.user_id,
new_user.session.user_id,
&object_id,
updates,
w_txn,
);
}
/// Migration the collab objects of the old user to new user. Currently, it only happens when
/// the user is a local user and try to use AppFlowy cloud service.
pub fn migration_user_to_cloud(
old_user: &UserMigrationContext,
old_collab_db: &Arc<RocksCollabDB>,
new_user: &UserMigrationContext,
new_collab_db: &Arc<RocksCollabDB>,
) -> FlowyResult<Option<FolderData>> {
let mut folder_data = None;
new_collab_db
.with_write_txn(|w_txn| {
let read_txn = old_collab_db.read_txn();
if let Ok(object_ids) = read_txn.get_all_docs() {
// Migration of all objects
for object_id in object_ids {
tracing::debug!("migrate object: {:?}", object_id);
if let Ok(updates) = read_txn.get_all_updates(old_user.session.user_id, &object_id) {
// If the object is a folder, migrate the folder data
if object_id == old_user.session.user_workspace.id {
folder_data = migrate_folder(
old_user.session.user_id,
&object_id,
&new_user.session.user_workspace.id,
updates,
);
} else if object_id == old_user.session.user_workspace.database_storage_id {
migrate_database_storage(
old_user.session.user_id,
&object_id,
new_user.session.user_id,
&new_user.session.user_workspace.database_storage_id,
updates,
w_txn,
);
} else {
migrate_object(
old_user.session.user_id,
new_user.session.user_id,
&object_id,
updates,
w_txn,
);
}
}
}
Ok(())
})
.map_err(|err| FlowyError::new(ErrorCode::Internal, err))?;
Ok(folder_data)
}
}
Ok(())
})
.map_err(|err| FlowyError::new(ErrorCode::Internal, err))?;
Ok(folder_data)
}
fn migrate_database_storage<'a, W>(

104
frontend/rust-lib/flowy-user/src/migrations/migration.rs Normal file
View File

@ -0,0 +1,104 @@
use crate::services::session_serde::Session;
use appflowy_integrate::RocksCollabDB;
use chrono::NaiveDateTime;
use diesel::{RunQueryDsl, SqliteConnection};
use flowy_error::FlowyResult;
use flowy_sqlite::schema::user_data_migration_records;
use flowy_sqlite::ConnectionPool;
use std::sync::Arc;
pub struct UserLocalDataMigration {
session: Session,
collab_db: Arc<RocksCollabDB>,
sqlite_pool: Arc<ConnectionPool>,
}
impl UserLocalDataMigration {
pub fn new(
session: Session,
collab_db: Arc<RocksCollabDB>,
sqlite_pool: Arc<ConnectionPool>,
) -> Self {
Self {
session,
collab_db,
sqlite_pool,
}
}
/// Executes a series of migrations.
///
/// This function applies each migration in the `migrations` vector that hasn't already been executed.
/// It retrieves the current migration records from the database, and for each migration in the `migrations` vector,
/// checks whether it has already been run. If it hasn't, the function runs the migration and adds it to the list of applied migrations.
///
/// The function does not apply a migration if its name is already in the list of applied migrations.
/// If a migration name is duplicated, the function logs an error message and continues with the next migration.
///
/// # Arguments
///
/// * `migrations` - A vector of boxed dynamic `UserDataMigration` objects representing the migrations to be applied.
///
pub fn run(self, migrations: Vec<Box<dyn UserDataMigration>>) -> FlowyResult<Vec<String>> {
let mut applied_migrations = vec![];
let conn = self.sqlite_pool.get()?;
let record = get_all_records(&*conn)?;
let mut duplicated_names = vec![];
for migration in migrations {
if record
.iter()
.find(|record| record.migration_name == migration.name())
.is_none()
{
let migration_name = migration.name().to_string();
if !duplicated_names.contains(&migration_name) {
migration.run(&self.session, &self.collab_db)?;
applied_migrations.push(migration.name().to_string());
save_record(&*conn, &migration_name);
duplicated_names.push(migration_name);
} else {
tracing::error!("Duplicated migration name: {}", migration_name);
}
}
}
Ok(applied_migrations)
}
}
pub trait UserDataMigration {
/// Migration with the same name will be skipped
fn name(&self) -> &str;
fn run(&self, user: &Session, collab_db: &Arc<RocksCollabDB>) -> FlowyResult<()>;
}
fn save_record(conn: &SqliteConnection, migration_name: &str) {
let new_record = NewUserDataMigrationRecord {
migration_name: migration_name.to_string(),
};
diesel::insert_into(user_data_migration_records::table)
.values(&new_record)
.execute(conn)
.expect("Error inserting new migration record");
}
fn get_all_records(conn: &SqliteConnection) -> FlowyResult<Vec<UserDataMigrationRecord>> {
Ok(
user_data_migration_records::table
.load::<UserDataMigrationRecord>(conn)
.unwrap_or_default(),
)
}
#[derive(Clone, Default, Queryable, Identifiable)]
#[table_name = "user_data_migration_records"]
pub struct UserDataMigrationRecord {
pub id: i32,
pub migration_name: String,
pub executed_at: NaiveDateTime,
}
#[derive(Insertable)]
#[table_name = "user_data_migration_records"]
pub struct NewUserDataMigrationRecord {
pub migration_name: String,
}

6
frontend/rust-lib/flowy-user/src/migrations/mod.rs Normal file
View File

@ -0,0 +1,6 @@
mod define;
pub mod historical_document;
pub mod local_user_to_cloud;
pub mod migration;
pub use define::*;

3
frontend/rust-lib/flowy-user/src/services/mod.rs
View File

@ -1,8 +1,7 @@
pub use user_session::*;
pub mod database;
mod session_serde;
mod user_data_migration;
pub mod session_serde;
mod user_session;
mod user_sql;
mod user_workspace_sql;

31
frontend/rust-lib/flowy-user/src/services/user_session.rs
View File

@ -20,9 +20,12 @@ use crate::entities::{UserProfilePB, UserSettingPB};
use crate::event_map::{
DefaultUserStatusCallback, SignUpContext, UserCloudServiceProvider, UserStatusCallback,
};
use crate::migrations::historical_document::HistoricalEmptyDocumentMigration;
use crate::migrations::local_user_to_cloud::migration_user_to_cloud;
use crate::migrations::migration::UserLocalDataMigration;
use crate::migrations::UserMigrationContext;
use crate::services::database::UserDB;
use crate::services::session_serde::Session;
use crate::services::user_data_migration::{UserDataMigration, UserMigrationContext};
use crate::services::user_sql::{UserTable, UserTableChangeset};
use crate::services::user_workspace_sql::UserWorkspaceTable;
use crate::{errors::FlowyError, notification::*};
@ -74,6 +77,25 @@ impl UserSession {
pub async fn init<C: UserStatusCallback + 'static>(&self, user_status_callback: C) {
if let Ok(session) = self.get_session() {
match (
self.database.get_collab_db(session.user_id),
self.database.get_pool(session.user_id),
) {
(Ok(collab_db), Ok(sqlite_pool)) => {
match UserLocalDataMigration::new(session.clone(), collab_db, sqlite_pool)
.run(vec![Box::new(HistoricalEmptyDocumentMigration)])
{
Ok(applied_migrations) => {
if applied_migrations.len() > 0 {
tracing::info!("Did apply migrations: {:?}", applied_migrations);
}
},
Err(e) => tracing::error!("User data migration failed: {:?}", e),
}
},
_ => tracing::error!("Failed to get collab db or sqlite pool"),
}
if let Err(e) = user_status_callback
.did_init(session.user_id, &session.user_workspace)
.await
@ -105,15 +127,14 @@ impl UserSession {
.map(|collab_db| Arc::downgrade(&collab_db))
}
pub async fn migrate_old_user_data(
async fn migrate_local_user_to_cloud(
&self,
old_user: &UserMigrationContext,
new_user: &UserMigrationContext,
) -> Result<Option<FolderData>, FlowyError> {
let old_collab_db = self.database.get_collab_db(old_user.session.user_id)?;
let new_collab_db = self.database.get_collab_db(new_user.session.user_id)?;
let folder_data =
UserDataMigration::migration(old_user, &old_collab_db, new_user, &new_collab_db)?;
let folder_data = migration_user_to_cloud(old_user, &old_collab_db, new_user, &new_collab_db)?;
Ok(folder_data)
}
@ -232,7 +253,7 @@ impl UserSession {
old_user.user_profile.id,
new_user.user_profile.id
);
match self.migrate_old_user_data(&old_user, &new_user).await {
match self.migrate_local_user_to_cloud(&old_user, &new_user).await {
Ok(folder_data) => sign_up_context.local_folder = folder_data,
Err(e) => tracing::error!("{:?}", e),
}