From b64da2c02f074d85565e8d5eb41c95ccae0dbe52 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Thu, 20 Jun 2024 07:44:57 +0800 Subject: [PATCH] refactor: File upload (#5542) * chore: rename service * refactor: upload * chore: save upload meta data * chore: add sql test * chore: uploader * chore: fix upload * chore: cache file and remove after finish * chore: retry upload * chore: pause when netowork unreachable * chore: add event test * chore: add test * chore: clippy * chore: update client-api commit id * chore: fix flutter test --- .../application/document_service.dart | 4 +- .../copy_and_paste/paste_from_image.dart | 7 - .../editor_plugins/image/image_util.dart | 4 +- frontend/appflowy_tauri/src-tauri/Cargo.lock | 68 +- frontend/appflowy_tauri/src-tauri/Cargo.toml | 2 +- frontend/appflowy_web/wasm-libs/Cargo.lock | 88 ++- frontend/appflowy_web/wasm-libs/Cargo.toml | 3 +- .../appflowy_web/wasm-libs/af-wasm/Cargo.toml | 1 - .../appflowy_web_app/src-tauri/Cargo.lock | 78 ++- .../appflowy_web_app/src-tauri/Cargo.toml | 2 +- frontend/rust-lib/Cargo.lock | 93 ++- frontend/rust-lib/Cargo.toml | 5 +- .../event-integration-test/Cargo.toml | 1 + .../event-integration-test/src/lib.rs | 54 +- .../af_cloud_test/file_upload_test.rs | 138 ++++ .../tests/document/af_cloud_test/mod.rs | 1 + .../tests/document/mod.rs | 11 +- frontend/rust-lib/flowy-core/Cargo.toml | 1 + .../src/deps_resolve/document_deps.rs | 4 +- .../src/deps_resolve/file_storage_deps.rs | 54 ++ .../flowy-core/src/deps_resolve/mod.rs | 1 + .../rust-lib/flowy-core/src/integrate/log.rs | 1 + .../flowy-core/src/integrate/trait_impls.rs | 79 ++- .../rust-lib/flowy-core/src/integrate/user.rs | 9 +- frontend/rust-lib/flowy-core/src/lib.rs | 16 +- frontend/rust-lib/flowy-document/Cargo.toml | 2 +- .../rust-lib/flowy-document/src/entities.rs | 7 +- .../flowy-document/src/event_handler.rs | 4 +- .../rust-lib/flowy-document/src/manager.rs | 75 +- .../flowy-document/tests/document/util.rs | 41 +- frontend/rust-lib/flowy-server/Cargo.toml | 1 + .../src/af_cloud/impls/file_storage.rs | 93 ++- .../flowy-server/src/af_cloud/server.rs | 4 +- .../flowy-server/src/local_server/server.rs | 4 +- frontend/rust-lib/flowy-server/src/server.rs | 4 +- .../src/supabase/file_storage/builder.rs | 7 +- .../src/supabase/file_storage/core.rs | 139 +--- .../src/supabase/file_storage/entities.rs | 3 +- .../src/supabase/file_storage/plan.rs | 2 +- .../flowy-server/src/supabase/server.rs | 38 +- frontend/rust-lib/flowy-server/tests/main.rs | 2 +- .../flowy-server/tests/supabase_test/util.rs | 21 +- .../2024-06-16-131359_file_upload/down.sql | 2 + .../2024-06-16-131359_file_upload/up.sql | 20 + frontend/rust-lib/flowy-sqlite/src/schema.rs | 38 +- .../rust-lib/flowy-storage-pub/Cargo.toml | 19 + .../flowy-storage-pub/src/chunked_byte.rs | 239 +++++++ .../rust-lib/flowy-storage-pub/src/cloud.rs | 176 +++++ .../rust-lib/flowy-storage-pub/src/lib.rs | 3 + .../rust-lib/flowy-storage-pub/src/storage.rs | 53 ++ frontend/rust-lib/flowy-storage/Cargo.toml | 23 +- .../rust-lib/flowy-storage/src/file_cache.rs | 88 +++ frontend/rust-lib/flowy-storage/src/lib.rs | 158 +---- .../rust-lib/flowy-storage/src/manager.rs | 648 ++++++++++++++++++ .../rust-lib/flowy-storage/src/native/mod.rs | 34 - .../rust-lib/flowy-storage/src/sqlite_sql.rs | 161 +++++ .../rust-lib/flowy-storage/src/uploader.rs | 294 ++++++++ .../rust-lib/flowy-storage/src/wasm/mod.rs | 12 - .../tests/multiple_part_upload_test.rs | 181 +++++ .../src/services/authenticate_user.rs | 4 + .../flowy-user/src/services/entities.rs | 5 + 61 files changed, 2687 insertions(+), 643 deletions(-) create mode 100644 frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/file_upload_test.rs create mode 100644 frontend/rust-lib/flowy-core/src/deps_resolve/file_storage_deps.rs create mode 100644 frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/down.sql create mode 100644 frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/up.sql create mode 100644 frontend/rust-lib/flowy-storage-pub/Cargo.toml create mode 100644 frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs create mode 100644 frontend/rust-lib/flowy-storage-pub/src/cloud.rs create mode 100644 frontend/rust-lib/flowy-storage-pub/src/lib.rs create mode 100644 frontend/rust-lib/flowy-storage-pub/src/storage.rs create mode 100644 frontend/rust-lib/flowy-storage/src/file_cache.rs create mode 100644 frontend/rust-lib/flowy-storage/src/manager.rs delete mode 100644 frontend/rust-lib/flowy-storage/src/native/mod.rs create mode 100644 frontend/rust-lib/flowy-storage/src/sqlite_sql.rs create mode 100644 frontend/rust-lib/flowy-storage/src/uploader.rs delete mode 100644 frontend/rust-lib/flowy-storage/src/wasm/mod.rs create mode 100644 frontend/rust-lib/flowy-storage/tests/multiple_part_upload_test.rs diff --git a/frontend/appflowy_flutter/lib/plugins/document/application/document_service.dart b/frontend/appflowy_flutter/lib/plugins/document/application/document_service.dart index 6a0b79c90e..9765209a38 100644 --- a/frontend/appflowy_flutter/lib/plugins/document/application/document_service.dart +++ b/frontend/appflowy_flutter/lib/plugins/document/application/document_service.dart @@ -114,14 +114,14 @@ class DocumentService { /// Upload a file to the cloud storage. Future> uploadFile({ required String localFilePath, - bool isAsync = true, + required String documentId, }) async { final workspace = await FolderEventReadCurrentWorkspace().send(); return workspace.fold((l) async { final payload = UploadFileParamsPB( workspaceId: l.id, localFilePath: localFilePath, - isAsync: isAsync, + documentId: documentId, ); final result = await DocumentEventUploadFile(payload).send(); return result; diff --git a/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/copy_and_paste/paste_from_image.dart b/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/copy_and_paste/paste_from_image.dart index 4de0961a85..afa7ae5274 100644 --- a/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/copy_and_paste/paste_from_image.dart +++ b/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/copy_and_paste/paste_from_image.dart @@ -53,13 +53,6 @@ extension PasteFromImage on EditorState { await File(copyToPath).writeAsBytes(imageBytes); final String? path; - if (context.mounted) { - showSnackBarMessage( - context, - LocaleKeys.document_imageBlock_imageIsUploading.tr(), - ); - } - if (isLocalMode) { path = await saveImageToLocalStorage(copyToPath); } else { diff --git a/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/image/image_util.dart b/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/image/image_util.dart index 352a6c878e..0710ffccd1 100644 --- a/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/image/image_util.dart +++ b/frontend/appflowy_flutter/lib/plugins/document/presentation/editor_plugins/image/image_util.dart @@ -49,9 +49,11 @@ Future<(String? path, String? errorMessage)> saveImageToCloudStorage( ); } final documentService = DocumentService(); + Log.debug("Uploading image local path: $localImagePath"); final result = await documentService.uploadFile( localFilePath: localImagePath, - isAsync: false, + // TODO(lucas): replace with actual documentId + documentId: "temp", ); return result.fold( (s) async { diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.lock b/frontend/appflowy_tauri/src-tauri/Cargo.lock index 5f50aa7f0d..aa07be5eeb 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.lock +++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock @@ -172,7 +172,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -192,7 +192,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bytes", @@ -772,7 +772,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "again", "anyhow", @@ -782,17 +782,16 @@ dependencies = [ "brotli", "bytes", "chrono", + "client-api-entity", "client-websocket", "collab", - "collab-entity", "collab-rt-entity", "collab-rt-protocol", - "database-entity", "futures-core", "futures-util", "getrandom 0.2.10", "gotrue", - "gotrue-entity", + "infra", "mime", "parking_lot 0.12.1", "prost", @@ -816,10 +815,22 @@ dependencies = [ "yrs", ] +[[package]] +name = "client-api-entity" +version = "0.1.0" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" +dependencies = [ + "collab-entity", + "collab-rt-entity", + "database-entity", + "gotrue-entity", + "shared-entity", +] + [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "futures-channel", "futures-util", @@ -1059,7 +1070,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -1084,7 +1095,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "async-trait", @@ -1441,7 +1452,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -1933,6 +1944,7 @@ dependencies = [ "flowy-server-pub", "flowy-sqlite", "flowy-storage", + "flowy-storage-pub", "flowy-user", "flowy-user-pub", "futures", @@ -2058,7 +2070,7 @@ dependencies = [ "flowy-document-pub", "flowy-error", "flowy-notification", - "flowy-storage", + "flowy-storage-pub", "futures", "getrandom 0.2.10", "indexmap 2.1.0", @@ -2266,6 +2278,7 @@ dependencies = [ "flowy-search-pub", "flowy-server-pub", "flowy-storage", + "flowy-storage-pub", "flowy-user-pub", "futures", "futures-util", @@ -2324,14 +2337,16 @@ dependencies = [ name = "flowy-storage" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", + "chrono", "flowy-error", + "flowy-sqlite", + "flowy-storage-pub", "fxhash", "lib-infra", - "mime", "mime_guess", - "reqwest", "serde", "serde_json", "tokio", @@ -2339,6 +2354,23 @@ dependencies = [ "url", ] +[[package]] +name = "flowy-storage-pub" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "client-api-entity", + "flowy-error", + "lib-infra", + "mime", + "mime_guess", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "flowy-user" version = "0.1.0" @@ -2862,7 +2894,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "futures-util", @@ -2879,7 +2911,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -3311,12 +3343,14 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", + "bytes", "reqwest", "serde", "serde_json", + "tokio", "tracing", ] @@ -5801,7 +5835,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.toml b/frontend/appflowy_tauri/src-tauri/Cargo.toml index 431a37d895..bae84180da 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.toml +++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml @@ -52,7 +52,7 @@ collab-user = { version = "0.2" } # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6262816043efeede8823d7a7ea252083adf407e9" } [dependencies] serde_json.workspace = true diff --git a/frontend/appflowy_web/wasm-libs/Cargo.lock b/frontend/appflowy_web/wasm-libs/Cargo.lock index 9354bd8418..42fefea9ac 100644 --- a/frontend/appflowy_web/wasm-libs/Cargo.lock +++ b/frontend/appflowy_web/wasm-libs/Cargo.lock @@ -122,7 +122,6 @@ dependencies = [ "flowy-notification", "flowy-server", "flowy-server-pub", - "flowy-storage", "flowy-user-pub", "js-sys", "lazy_static", @@ -216,7 +215,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -236,7 +235,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bytes", @@ -562,7 +561,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "again", "anyhow", @@ -572,17 +571,16 @@ dependencies = [ "brotli", "bytes", "chrono", + "client-api-entity", "client-websocket", "collab", - "collab-entity", "collab-rt-entity", "collab-rt-protocol", - "database-entity", "futures-core", "futures-util", "getrandom 0.2.12", "gotrue", - "gotrue-entity", + "infra", "mime", "parking_lot 0.12.1", "prost", @@ -606,10 +604,22 @@ dependencies = [ "yrs", ] +[[package]] +name = "client-api-entity" +version = "0.1.0" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" +dependencies = [ + "collab-entity", + "collab-rt-entity", + "database-entity", + "gotrue-entity", + "shared-entity", +] + [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "futures-channel", "futures-util", @@ -787,7 +797,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -812,7 +822,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "async-trait", @@ -981,7 +991,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.8.0", + "phf 0.11.2", "smallvec", ] @@ -1026,7 +1036,7 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -1380,7 +1390,7 @@ dependencies = [ "flowy-document-pub", "flowy-error", "flowy-notification", - "flowy-storage", + "flowy-storage-pub", "futures", "getrandom 0.2.12", "indexmap", @@ -1438,8 +1448,10 @@ dependencies = [ "fancy-regex 0.11.0", "flowy-codegen", "flowy-derive", + "flowy-sqlite", "lib-dispatch", "protobuf", + "r2d2", "reqwest", "serde", "serde_json", @@ -1550,6 +1562,7 @@ dependencies = [ "flowy-search-pub", "flowy-server-pub", "flowy-storage", + "flowy-storage-pub", "flowy-user-pub", "futures", "futures-util", @@ -1608,14 +1621,16 @@ dependencies = [ name = "flowy-storage" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", + "chrono", "flowy-error", + "flowy-sqlite", + "flowy-storage-pub", "fxhash", "lib-infra", - "mime", "mime_guess", - "reqwest", "serde", "serde_json", "tokio", @@ -1623,6 +1638,23 @@ dependencies = [ "url", ] +[[package]] +name = "flowy-storage-pub" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "client-api-entity", + "flowy-error", + "lib-infra", + "mime", + "mime_guess", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "flowy-user-pub" version = "0.1.0" @@ -1887,7 +1919,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "futures-util", @@ -1904,7 +1936,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -2205,12 +2237,14 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", + "bytes", "reqwest", "serde", "serde_json", + "tokio", "tracing", ] @@ -2932,7 +2966,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" dependencies = [ - "phf_macros", + "phf_macros 0.8.0", "phf_shared 0.8.0", "proc-macro-hack", ] @@ -2952,6 +2986,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ + "phf_macros 0.11.2", "phf_shared 0.11.2", ] @@ -3019,6 +3054,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "phf_shared" version = "0.8.0" @@ -3903,7 +3951,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", diff --git a/frontend/appflowy_web/wasm-libs/Cargo.toml b/frontend/appflowy_web/wasm-libs/Cargo.toml index 24b745fd0a..9816e6ef94 100644 --- a/frontend/appflowy_web/wasm-libs/Cargo.toml +++ b/frontend/appflowy_web/wasm-libs/Cargo.toml @@ -20,7 +20,6 @@ flowy-derive = { path = "../../rust-lib/build-tool/flowy-derive" } flowy-codegen = { path = "../../rust-lib/build-tool/flowy-codegen" } flowy-document = { path = "../../rust-lib/flowy-document" } flowy-folder = { path = "../../rust-lib/flowy-folder" } -flowy-storage = { path = "../../rust-lib/flowy-storage" } lib-infra = { path = "../../rust-lib/lib-infra" } bytes = { version = "1.5" } protobuf = { version = "2.28.0" } @@ -55,7 +54,7 @@ yrs = "0.18.8" # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6262816043efeede8823d7a7ea252083adf407e9" } [profile.dev] opt-level = 0 diff --git a/frontend/appflowy_web/wasm-libs/af-wasm/Cargo.toml b/frontend/appflowy_web/wasm-libs/af-wasm/Cargo.toml index db754e681e..c504233088 100644 --- a/frontend/appflowy_web/wasm-libs/af-wasm/Cargo.toml +++ b/frontend/appflowy_web/wasm-libs/af-wasm/Cargo.toml @@ -21,7 +21,6 @@ tokio-stream.workspace = true af-user.workspace = true af-persistence.workspace = true -flowy-storage = { workspace = true } flowy-notification = { workspace = true, features = ["web_ts"] } flowy-user-pub = { workspace = true } flowy-server = { workspace = true } diff --git a/frontend/appflowy_web_app/src-tauri/Cargo.lock b/frontend/appflowy_web_app/src-tauri/Cargo.lock index e3589d8718..a4a230c514 100644 --- a/frontend/appflowy_web_app/src-tauri/Cargo.lock +++ b/frontend/appflowy_web_app/src-tauri/Cargo.lock @@ -156,14 +156,14 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -183,7 +183,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bytes", @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "again", "anyhow", @@ -756,17 +756,16 @@ dependencies = [ "brotli", "bytes", "chrono", + "client-api-entity", "client-websocket", "collab", - "collab-entity", "collab-rt-entity", "collab-rt-protocol", - "database-entity", "futures-core", "futures-util", "getrandom 0.2.12", "gotrue", - "gotrue-entity", + "infra", "mime", "parking_lot 0.12.1", "prost", @@ -790,10 +789,22 @@ dependencies = [ "yrs", ] +[[package]] +name = "client-api-entity" +version = "0.1.0" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" +dependencies = [ + "collab-entity", + "collab-rt-entity", + "database-entity", + "gotrue-entity", + "shared-entity", +] + [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "futures-channel", "futures-util", @@ -1042,7 +1053,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -1067,7 +1078,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "async-trait", @@ -1317,7 +1328,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa 1.0.10", - "phf 0.11.2", + "phf 0.8.0", "smallvec", ] @@ -1428,7 +1439,7 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -1970,6 +1981,7 @@ dependencies = [ "flowy-server-pub", "flowy-sqlite", "flowy-storage", + "flowy-storage-pub", "flowy-user", "flowy-user-pub", "futures", @@ -2095,7 +2107,7 @@ dependencies = [ "flowy-document-pub", "flowy-error", "flowy-notification", - "flowy-storage", + "flowy-storage-pub", "futures", "getrandom 0.2.12", "indexmap 2.2.6", @@ -2303,6 +2315,7 @@ dependencies = [ "flowy-search-pub", "flowy-server-pub", "flowy-storage", + "flowy-storage-pub", "flowy-user-pub", "futures", "futures-util", @@ -2361,14 +2374,16 @@ dependencies = [ name = "flowy-storage" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", + "chrono", "flowy-error", + "flowy-sqlite", + "flowy-storage-pub", "fxhash", "lib-infra", - "mime", "mime_guess", - "reqwest", "serde", "serde_json", "tokio", @@ -2376,6 +2391,23 @@ dependencies = [ "url", ] +[[package]] +name = "flowy-storage-pub" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "client-api-entity", + "flowy-error", + "lib-infra", + "mime", + "mime_guess", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "flowy-user" version = "0.1.0" @@ -2936,7 +2968,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "futures-util", @@ -2953,7 +2985,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -3390,12 +3422,14 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", + "bytes", "reqwest", "serde", "serde_json", + "tokio", "tracing", ] @@ -4897,7 +4931,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4918,7 +4952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.55", @@ -5896,7 +5930,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", diff --git a/frontend/appflowy_web_app/src-tauri/Cargo.toml b/frontend/appflowy_web_app/src-tauri/Cargo.toml index 597dfb2fb4..ddf01dabc5 100644 --- a/frontend/appflowy_web_app/src-tauri/Cargo.toml +++ b/frontend/appflowy_web_app/src-tauri/Cargo.toml @@ -52,7 +52,7 @@ collab-user = { version = "0.2" } # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6262816043efeede8823d7a7ea252083adf407e9" } [dependencies] serde_json.workspace = true diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 71fa48ce99..ce749cbe0c 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -163,7 +163,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -183,7 +183,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bytes", @@ -664,7 +664,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "again", "anyhow", @@ -674,17 +674,16 @@ dependencies = [ "brotli", "bytes", "chrono", + "client-api-entity", "client-websocket", "collab", - "collab-entity", "collab-rt-entity", "collab-rt-protocol", - "database-entity", "futures-core", "futures-util", "getrandom 0.2.10", "gotrue", - "gotrue-entity", + "infra", "mime", "parking_lot 0.12.1", "prost", @@ -708,10 +707,22 @@ dependencies = [ "yrs", ] +[[package]] +name = "client-api-entity" +version = "0.1.0" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" +dependencies = [ + "collab-entity", + "collab-rt-entity", + "database-entity", + "gotrue-entity", + "shared-entity", +] + [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "futures-channel", "futures-util", @@ -920,7 +931,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "bincode", @@ -945,7 +956,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "async-trait", @@ -1165,7 +1176,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.11.2", + "phf 0.8.0", "smallvec", ] @@ -1265,7 +1276,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -1519,6 +1530,7 @@ dependencies = [ "flowy-server", "flowy-server-pub", "flowy-storage", + "flowy-storage-pub", "flowy-user", "flowy-user-pub", "futures", @@ -1753,6 +1765,7 @@ dependencies = [ "flowy-server-pub", "flowy-sqlite", "flowy-storage", + "flowy-storage-pub", "flowy-user", "flowy-user-pub", "futures", @@ -1879,7 +1892,7 @@ dependencies = [ "flowy-document-pub", "flowy-error", "flowy-notification", - "flowy-storage", + "flowy-storage-pub", "futures", "getrandom 0.2.10", "indexmap 2.1.0", @@ -2092,6 +2105,7 @@ dependencies = [ "flowy-search-pub", "flowy-server-pub", "flowy-storage", + "flowy-storage-pub", "flowy-user-pub", "futures", "futures-util", @@ -2154,19 +2168,40 @@ dependencies = [ name = "flowy-storage" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", + "chrono", "flowy-error", + "flowy-sqlite", + "flowy-storage-pub", "fxhash", "lib-infra", - "mime", "mime_guess", - "reqwest", + "rand 0.8.5", "serde", "serde_json", "tokio", "tracing", "url", + "uuid", +] + +[[package]] +name = "flowy-storage-pub" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "client-api-entity", + "flowy-error", + "lib-infra", + "mime", + "mime_guess", + "serde", + "serde_json", + "tokio", ] [[package]] @@ -2532,7 +2567,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "futures-util", @@ -2549,7 +2584,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", @@ -2914,12 +2949,14 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", + "bytes", "reqwest", "serde", "serde_json", + "tokio", "tracing", ] @@ -3790,7 +3827,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" dependencies = [ - "phf_macros 0.8.0", + "phf_macros", "phf_shared 0.8.0", "proc-macro-hack", ] @@ -3810,7 +3847,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ - "phf_macros 0.11.2", "phf_shared 0.11.2", ] @@ -3878,19 +3914,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "phf_macros" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" -dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", - "proc-macro2", - "quote", - "syn 2.0.47", -] - [[package]] name = "phf_shared" version = "0.8.0" @@ -4094,7 +4117,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4115,7 +4138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.47", @@ -5012,7 +5035,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=430e3e15c9a1dc6aba2a9599d17d946a61ac7cae#430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6262816043efeede8823d7a7ea252083adf407e9#6262816043efeede8823d7a7ea252083adf407e9" dependencies = [ "anyhow", "app-error", diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml index f561310fec..2734218319 100644 --- a/frontend/rust-lib/Cargo.toml +++ b/frontend/rust-lib/Cargo.toml @@ -31,6 +31,7 @@ members = [ "flowy-search-pub", "flowy-chat", "flowy-chat-pub", + "flowy-storage-pub", ] resolver = "2" @@ -59,6 +60,7 @@ flowy-server-pub = { workspace = true, path = "flowy-server-pub" } flowy-config = { workspace = true, path = "flowy-config" } flowy-encrypt = { workspace = true, path = "flowy-encrypt" } flowy-storage = { workspace = true, path = "flowy-storage" } +flowy-storage-pub = { workspace = true, path = "flowy-storage-pub" } flowy-search = { workspace = true, path = "flowy-search" } flowy-search-pub = { workspace = true, path = "flowy-search-pub" } collab-integrate = { workspace = true, path = "collab-integrate" } @@ -95,7 +97,8 @@ validator = { version = "0.16.1", features = ["derive"] } # Run the script.add_workspace_members: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "430e3e15c9a1dc6aba2a9599d17d946a61ac7cae" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6262816043efeede8823d7a7ea252083adf407e9" } +client-api-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6262816043efeede8823d7a7ea252083adf407e9" } [profile.dev] opt-level = 1 diff --git a/frontend/rust-lib/event-integration-test/Cargo.toml b/frontend/rust-lib/event-integration-test/Cargo.toml index aca158bfaf..a644709791 100644 --- a/frontend/rust-lib/event-integration-test/Cargo.toml +++ b/frontend/rust-lib/event-integration-test/Cargo.toml @@ -24,6 +24,7 @@ flowy-server-pub = { workspace = true } flowy-notification = { workspace = true } anyhow.workspace = true flowy-storage = { workspace = true } +flowy-storage-pub = { workspace = true } flowy-search = { workspace = true } semver = "1.0.23" diff --git a/frontend/rust-lib/event-integration-test/src/lib.rs b/frontend/rust-lib/event-integration-test/src/lib.rs index 2ae16c74b3..69096bbd37 100644 --- a/frontend/rust-lib/event-integration-test/src/lib.rs +++ b/frontend/rust-lib/event-integration-test/src/lib.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use std::time::Duration; use nanoid::nanoid; -use parking_lot::RwLock; +use parking_lot::{Mutex, RwLock}; use semver::Version; use tokio::select; use tokio::time::sleep; @@ -37,7 +37,7 @@ pub struct EventIntegrationTest { pub authenticator: Arc>, pub appflowy_core: AppFlowyCore, #[allow(dead_code)] - cleaner: Arc, + cleaner: Arc>, pub notification_sender: TestNotificationSender, } @@ -52,10 +52,26 @@ impl EventIntegrationTest { Self::new_with_user_data_path(temp_dir, name.to_string()).await } + pub async fn new_with_config(config: AppFlowyCoreConfig) -> Self { + let clean_path = config.storage_path.clone(); + let inner = init_core(config).await; + let notification_sender = TestNotificationSender::new(); + let authenticator = Arc::new(RwLock::new(AuthenticatorPB::Local)); + register_notification_sender(notification_sender.clone()); + + // In case of dropping the runtime that runs the core, we need to forget the dispatcher + std::mem::forget(inner.dispatcher()); + Self { + appflowy_core: inner, + authenticator, + notification_sender, + cleaner: Arc::new(Mutex::new(Cleaner::new(PathBuf::from(clean_path)))), + } + } + pub async fn new_with_user_data_path(path_buf: PathBuf, name: String) -> Self { let path = path_buf.to_str().unwrap().to_string(); let device_id = uuid::Uuid::new_v4().to_string(); - let config = AppFlowyCoreConfig::new( Version::new(0, 5, 8), path.clone(), @@ -72,20 +88,11 @@ impl EventIntegrationTest { // "lib_dispatch".to_string(), ], ); + Self::new_with_config(config).await + } - let inner = init_core(config).await; - let notification_sender = TestNotificationSender::new(); - let authenticator = Arc::new(RwLock::new(AuthenticatorPB::Local)); - register_notification_sender(notification_sender.clone()); - - // In case of dropping the runtime that runs the core, we need to forget the dispatcher - std::mem::forget(inner.dispatcher()); - Self { - appflowy_core: inner, - authenticator, - notification_sender, - cleaner: Arc::new(Cleaner(path_buf)), - } + pub fn set_no_cleanup(&mut self) { + self.cleaner.lock().should_clean = false; } pub fn instance_name(&self) -> String { @@ -173,11 +180,18 @@ impl std::ops::Deref for EventIntegrationTest { } } -pub struct Cleaner(PathBuf); +#[derive(Clone)] +pub struct Cleaner { + dir: PathBuf, + should_clean: bool, +} impl Cleaner { pub fn new(dir: PathBuf) -> Self { - Cleaner(dir) + Self { + dir, + should_clean: true, + } } fn cleanup(dir: &PathBuf) { @@ -187,6 +201,8 @@ impl Cleaner { impl Drop for Cleaner { fn drop(&mut self) { - Self::cleanup(&self.0) + if self.should_clean { + Self::cleanup(&self.dir) + } } } diff --git a/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/file_upload_test.rs b/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/file_upload_test.rs new file mode 100644 index 0000000000..4b2702469c --- /dev/null +++ b/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/file_upload_test.rs @@ -0,0 +1,138 @@ +use crate::document::generate_random_bytes; +use event_integration_test::user_event::user_localhost_af_cloud; +use event_integration_test::EventIntegrationTest; +use flowy_storage_pub::storage::UploadStatus; +use std::env::temp_dir; +use std::time::Duration; +use tokio::fs; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; + +#[tokio::test] +async fn af_cloud_upload_file_test() { + user_localhost_af_cloud().await; + let test = EventIntegrationTest::new().await; + test.af_cloud_sign_up().await; + + let workspace_id = test.get_current_workspace().await.id; + let file_path = generate_file_with_bytes_len(1024).await.0; + let mut rx = test.storage_manager.subscribe_upload_result(); + + let created_upload = test + .storage_manager + .storage_service + .create_upload(&workspace_id, "temp_test", &file_path) + .await + .unwrap(); + + while let Ok(result) = rx.recv().await { + if result.file_id == created_upload.file_id { + if result.status == UploadStatus::Finish { + break; + } + } + } + + let _ = fs::remove_file(file_path).await; +} + +#[tokio::test] +async fn af_cloud_upload_big_file_test() { + user_localhost_af_cloud().await; + let mut test = EventIntegrationTest::new().await; + test.af_cloud_sign_up().await; + tokio::time::sleep(Duration::from_secs(6)).await; + + let workspace_id = test.get_current_workspace().await.id; + let (file_path, upload_data) = generate_file_with_bytes_len(30 * 1024 * 1024).await; + let created_upload = test + .storage_manager + .storage_service + .create_upload(&workspace_id, "temp_test", &file_path) + .await + .unwrap(); + + let mut rx = test.storage_manager.subscribe_upload_result(); + while let Ok(result) = rx.recv().await { + if result.file_id == created_upload.file_id { + if result.status == UploadStatus::InProgress { + break; + } + } + } + + // Simulate a restart + let config = test.config.clone(); + test.set_no_cleanup(); + drop(test); + tokio::time::sleep(Duration::from_secs(3)).await; + + // Restart the test. It will load unfinished uploads + let test = EventIntegrationTest::new_with_config(config).await; + let mut rx = test.storage_manager.subscribe_upload_result(); + while let Ok(result) = rx.recv().await { + if result.file_id == created_upload.file_id { + if result.status == UploadStatus::Finish { + break; + } + } + } + + // download the file and then compare the data. + let file_service = test + .server_provider + .get_server() + .unwrap() + .file_storage() + .unwrap(); + let file = file_service.get_object(created_upload.url).await.unwrap(); + assert_eq!(file.raw.to_vec(), upload_data); + + let _ = fs::remove_file(file_path).await; +} + +#[tokio::test] +async fn af_cloud_upload_6_files_test() { + user_localhost_af_cloud().await; + let test = EventIntegrationTest::new().await; + test.af_cloud_sign_up().await; + + let workspace_id = test.get_current_workspace().await.id; + let mut rx = test.storage_manager.subscribe_upload_result(); + + let mut created_uploads = vec![]; + for file_size in vec![1, 2, 5, 8, 12, 20] { + let file_path = generate_file_with_bytes_len(file_size * 1024 * 1024) + .await + .0; + let created_upload = test + .storage_manager + .storage_service + .create_upload(&workspace_id, "temp_test", &file_path) + .await + .unwrap(); + created_uploads.push(created_upload); + + let _ = fs::remove_file(file_path).await; + } + + while let Ok(result) = rx.recv().await { + if result.status == UploadStatus::Finish { + created_uploads.retain(|upload| upload.file_id != result.file_id); + } + + if created_uploads.is_empty() { + break; + } + } +} + +async fn generate_file_with_bytes_len(len: usize) -> (String, Vec) { + let data = generate_random_bytes(len); + let file_dir = temp_dir().join(uuid::Uuid::new_v4().to_string()); + let file_path = file_dir.to_str().unwrap().to_string(); + let mut file = File::create(file_dir).await.unwrap(); + file.write_all(&data).await.unwrap(); + + (file_path, data) +} diff --git a/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/mod.rs b/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/mod.rs index 0e50d38f75..c63deb8798 100644 --- a/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/mod.rs +++ b/frontend/rust-lib/event-integration-test/tests/document/af_cloud_test/mod.rs @@ -1 +1,2 @@ mod edit_test; +mod file_upload_test; diff --git a/frontend/rust-lib/event-integration-test/tests/document/mod.rs b/frontend/rust-lib/event-integration-test/tests/document/mod.rs index ba2833ee49..e11cb782f4 100644 --- a/frontend/rust-lib/event-integration-test/tests/document/mod.rs +++ b/frontend/rust-lib/event-integration-test/tests/document/mod.rs @@ -4,7 +4,7 @@ mod af_cloud_test; // #[cfg(feature = "supabase_cloud_test")] // mod supabase_test; -use rand::{distributions::Alphanumeric, Rng}; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; pub fn generate_random_string(len: usize) -> String { let rng = rand::thread_rng(); @@ -14,3 +14,12 @@ pub fn generate_random_string(len: usize) -> String { .map(char::from) .collect() } + +pub fn generate_random_bytes(size: usize) -> Vec { + let s: String = thread_rng() + .sample_iter(&Alphanumeric) + .take(size) + .map(char::from) + .collect(); + s.into_bytes() +} diff --git a/frontend/rust-lib/flowy-core/Cargo.toml b/frontend/rust-lib/flowy-core/Cargo.toml index 6cdcb0f5bd..3642847fa4 100644 --- a/frontend/rust-lib/flowy-core/Cargo.toml +++ b/frontend/rust-lib/flowy-core/Cargo.toml @@ -31,6 +31,7 @@ collab = { workspace = true } diesel.workspace = true uuid.workspace = true flowy-storage = { workspace = true } +flowy-storage-pub = { workspace = true } client-api.workspace = true flowy-chat = { workspace = true } flowy-chat-pub = { workspace = true } diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/document_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/document_deps.rs index 1876392eeb..a4203d8268 100644 --- a/frontend/rust-lib/flowy-core/src/deps_resolve/document_deps.rs +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/document_deps.rs @@ -8,7 +8,7 @@ use flowy_document::entities::{DocumentSnapshotData, DocumentSnapshotMeta}; use flowy_document::manager::{DocumentManager, DocumentSnapshotService, DocumentUserService}; use flowy_document_pub::cloud::DocumentCloudService; use flowy_error::{FlowyError, FlowyResult}; -use flowy_storage::ObjectStorageService; +use flowy_storage_pub::storage::StorageService; use flowy_user::services::authenticate_user::AuthenticateUser; pub struct DocumentDepsResolver(); @@ -18,7 +18,7 @@ impl DocumentDepsResolver { _database_manager: &Arc, collab_builder: Arc, cloud_service: Arc, - storage_service: Weak, + storage_service: Weak, ) -> Arc { let user_service: Arc = Arc::new(DocumentUserImpl(authenticate_user.clone())); diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/file_storage_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/file_storage_deps.rs new file mode 100644 index 0000000000..f0e6985a78 --- /dev/null +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/file_storage_deps.rs @@ -0,0 +1,54 @@ +use flowy_error::FlowyError; +use flowy_sqlite::DBConnection; +use flowy_storage::manager::{StorageManager, StorageUserService}; +use flowy_storage_pub::cloud::StorageCloudService; +use flowy_user::services::authenticate_user::AuthenticateUser; +use std::sync::{Arc, Weak}; + +pub struct FileStorageResolver; + +impl FileStorageResolver { + pub fn resolve( + authenticate_user: Weak, + cloud_service: Arc, + root: &str, + ) -> Arc { + let user_service = FileStorageServiceImpl { + user: authenticate_user, + root_dir: root.to_owned(), + }; + Arc::new(StorageManager::new(cloud_service, Arc::new(user_service))) + } +} + +struct FileStorageServiceImpl { + user: Weak, + root_dir: String, +} +impl FileStorageServiceImpl { + fn upgrade_user(&self) -> Result, FlowyError> { + let user = self + .user + .upgrade() + .ok_or(FlowyError::internal().with_context("Unexpected error: UserSession is None"))?; + Ok(user) + } +} + +impl StorageUserService for FileStorageServiceImpl { + fn user_id(&self) -> Result { + self.upgrade_user()?.user_id() + } + + fn workspace_id(&self) -> Result { + self.upgrade_user()?.workspace_id() + } + + fn sqlite_connection(&self, uid: i64) -> Result { + self.upgrade_user()?.get_sqlite_connection(uid) + } + + fn get_application_root_dir(&self) -> &str { + &self.root_dir + } +} diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs index a75589e89e..2b2bc31118 100644 --- a/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs @@ -12,5 +12,6 @@ mod folder_deps; mod chat_deps; mod database_deps; +pub mod file_storage_deps; mod search_deps; mod user_deps; diff --git a/frontend/rust-lib/flowy-core/src/integrate/log.rs b/frontend/rust-lib/flowy-core/src/integrate/log.rs index c855a557ac..ddf8d32277 100644 --- a/frontend/rust-lib/flowy-core/src/integrate/log.rs +++ b/frontend/rust-lib/flowy-core/src/integrate/log.rs @@ -53,6 +53,7 @@ pub fn create_log_filter(level: String, with_crates: Vec, platform: Plat filters.push(format!("lib_infra={}", level)); filters.push(format!("flowy_search={}", level)); filters.push(format!("flowy_chat={}", level)); + filters.push(format!("flowy_storage={}", level)); // Enable the frontend logs. DO NOT DISABLE. // These logs are essential for debugging and verifying frontend behavior. filters.push(format!("dart_ffi={}", level)); diff --git a/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs b/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs index 1853fd12d3..8228531d38 100644 --- a/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs +++ b/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs @@ -1,6 +1,5 @@ use client_api::entity::search_dto::SearchDocumentResponseItem; use flowy_search_pub::cloud::SearchCloudService; -use flowy_storage::{ObjectIdentity, ObjectStorageService}; use std::sync::Arc; use anyhow::Error; @@ -28,13 +27,14 @@ use flowy_database_pub::cloud::{ }; use flowy_document::deps::DocumentData; use flowy_document_pub::cloud::{DocumentCloudService, DocumentSnapshot}; -use flowy_error::FlowyError; +use flowy_error::{FlowyError, FlowyResult}; use flowy_folder_pub::cloud::{ FolderCloudService, FolderCollabParams, FolderData, FolderSnapshot, Workspace, WorkspaceRecord, }; use flowy_server_pub::af_cloud_config::AFCloudConfiguration; use flowy_server_pub::supabase_config::SupabaseConfiguration; -use flowy_storage::ObjectValue; +use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService}; +use flowy_storage_pub::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse}; use flowy_user_pub::cloud::{UserCloudService, UserCloudServiceProvider}; use flowy_user_pub::entities::{Authenticator, UserTokenState}; use lib_infra::async_trait::async_trait; @@ -42,7 +42,8 @@ use lib_infra::future::FutureResult; use crate::integrate::server::{Server, ServerProvider}; -impl ObjectStorageService for ServerProvider { +#[async_trait] +impl StorageCloudService for ServerProvider { fn get_object_url(&self, object_id: ObjectIdentity) -> FutureResult { let server = self.get_server(); FutureResult::new(async move { @@ -59,21 +60,85 @@ impl ObjectStorageService for ServerProvider { }) } - fn delete_object(&self, url: String) -> FutureResult<(), FlowyError> { + fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError> { let server = self.get_server(); + let url = url.to_string(); FutureResult::new(async move { let storage = server?.file_storage().ok_or(FlowyError::internal())?; - storage.delete_object(url).await + storage.delete_object(&url).await }) } - fn get_object(&self, url: String) -> FutureResult { + fn get_object(&self, url: String) -> FutureResult { let server = self.get_server(); FutureResult::new(async move { let storage = server?.file_storage().ok_or(FlowyError::internal())?; storage.get_object(url).await }) } + + fn get_object_url_v1( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + ) -> FlowyResult { + let server = self.get_server()?; + let storage = server.file_storage().ok_or(FlowyError::internal())?; + storage.get_object_url_v1(workspace_id, parent_dir, file_id) + } + + async fn create_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + content_type: &str, + ) -> Result { + let server = self.get_server(); + let storage = server?.file_storage().ok_or(FlowyError::internal())?; + storage + .create_upload(workspace_id, parent_dir, file_id, content_type) + .await + } + + async fn upload_part( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + part_number: i32, + body: Vec, + ) -> Result { + let server = self.get_server(); + let storage = server?.file_storage().ok_or(FlowyError::internal())?; + storage + .upload_part( + workspace_id, + parent_dir, + upload_id, + file_id, + part_number, + body, + ) + .await + } + + async fn complete_upload( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + parts: Vec, + ) -> Result<(), FlowyError> { + let server = self.get_server(); + let storage = server?.file_storage().ok_or(FlowyError::internal())?; + storage + .complete_upload(workspace_id, parent_dir, upload_id, file_id, parts) + .await + } } impl UserCloudServiceProvider for ServerProvider { diff --git a/frontend/rust-lib/flowy-core/src/integrate/user.rs b/frontend/rust-lib/flowy-core/src/integrate/user.rs index f8219cdaff..5288a0ee24 100644 --- a/frontend/rust-lib/flowy-core/src/integrate/user.rs +++ b/frontend/rust-lib/flowy-core/src/integrate/user.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::Context; -use tracing::event; +use tracing::{event, trace}; use collab_entity::CollabType; use collab_integrate::collab_builder::AppFlowyCollabBuilder; @@ -9,13 +9,13 @@ use flowy_database2::DatabaseManager; use flowy_document::manager::DocumentManager; use flowy_error::{FlowyError, FlowyResult}; use flowy_folder::manager::{FolderInitDataSource, FolderManager}; +use flowy_storage::manager::StorageManager; use flowy_user::event_map::UserStatusCallback; use flowy_user_pub::cloud::{UserCloudConfig, UserCloudServiceProvider}; use flowy_user_pub::entities::{Authenticator, UserProfile, UserWorkspace}; use lib_infra::future::{to_fut, Fut}; use crate::integrate::server::{Server, ServerProvider}; -use crate::AppFlowyCoreConfig; pub(crate) struct UserStatusCallbackImpl { pub(crate) collab_builder: Arc, @@ -23,8 +23,7 @@ pub(crate) struct UserStatusCallbackImpl { pub(crate) database_manager: Arc, pub(crate) document_manager: Arc, pub(crate) server_provider: Arc, - #[allow(dead_code)] - pub(crate) config: AppFlowyCoreConfig, + pub(crate) storage_manager: Arc, } impl UserStatusCallback for UserStatusCallbackImpl { @@ -213,6 +212,8 @@ impl UserStatusCallback for UserStatusCallbackImpl { } fn did_update_network(&self, reachable: bool) { + trace!("Notify did update network: reachable: {}", reachable); self.collab_builder.update_network(reachable); + self.storage_manager.update_network_reachable(reachable); } } diff --git a/frontend/rust-lib/flowy-core/src/lib.rs b/frontend/rust-lib/flowy-core/src/lib.rs index 818fdfe356..11bba9de8c 100644 --- a/frontend/rust-lib/flowy-core/src/lib.rs +++ b/frontend/rust-lib/flowy-core/src/lib.rs @@ -2,7 +2,6 @@ use flowy_search::folder::indexer::FolderIndexManagerImpl; use flowy_search::services::manager::SearchManager; -use flowy_storage::ObjectStorageService; use std::sync::{Arc, Weak}; use std::time::Duration; use sysinfo::System; @@ -18,6 +17,7 @@ use flowy_folder::manager::FolderManager; use flowy_server::af_cloud::define::ServerUser; use flowy_sqlite::kv::StorePreferences; +use flowy_storage::manager::StorageManager; use flowy_user::services::authenticate_user::AuthenticateUser; use flowy_user::services::entities::UserConfig; use flowy_user::user_manager::UserManager; @@ -30,6 +30,7 @@ use lib_log::stream_log::StreamLogSender; use module::make_plugins; use crate::config::AppFlowyCoreConfig; +use crate::deps_resolve::file_storage_deps::FileStorageResolver; use crate::deps_resolve::*; use crate::integrate::collab_interact::CollabInteractImpl; use crate::integrate::log::init_log; @@ -59,6 +60,7 @@ pub struct AppFlowyCore { pub store_preference: Arc, pub search_manager: Arc, pub chat_manager: Arc, + pub storage_manager: Arc, } impl AppFlowyCore { @@ -140,7 +142,13 @@ impl AppFlowyCore { collab_builder, search_manager, chat_manager, + storage_manager, ) = async { + let storage_manager = FileStorageResolver::resolve( + Arc::downgrade(&authenticate_user), + server_provider.clone(), + &user_config.storage_path, + ); /// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded /// on demand based on the [CollabPluginConfig]. let collab_builder = Arc::new(AppFlowyCollabBuilder::new( @@ -164,7 +172,7 @@ impl AppFlowyCore { &database_manager, collab_builder.clone(), server_provider.clone(), - Arc::downgrade(&(server_provider.clone() as Arc)), + Arc::downgrade(&storage_manager.storage_service), ); let chat_manager = @@ -216,6 +224,7 @@ impl AppFlowyCore { collab_builder, search_manager, chat_manager, + storage_manager, ) } .await; @@ -226,7 +235,7 @@ impl AppFlowyCore { database_manager: database_manager.clone(), document_manager: document_manager.clone(), server_provider: server_provider.clone(), - config: config.clone(), + storage_manager: storage_manager.clone(), }; let collab_interact_impl = CollabInteractImpl { @@ -267,6 +276,7 @@ impl AppFlowyCore { store_preference, search_manager, chat_manager, + storage_manager, } } diff --git a/frontend/rust-lib/flowy-document/Cargo.toml b/frontend/rust-lib/flowy-document/Cargo.toml index 1b2cf3c593..e9bc84b8fe 100644 --- a/frontend/rust-lib/flowy-document/Cargo.toml +++ b/frontend/rust-lib/flowy-document/Cargo.toml @@ -14,7 +14,7 @@ collab-entity = { workspace = true } collab-plugins = { workspace = true } collab-integrate = { workspace = true } flowy-document-pub = { workspace = true } -flowy-storage = { workspace = true } +flowy-storage-pub = { workspace = true } flowy-derive.workspace = true flowy-notification = { workspace = true } flowy-error = { path = "../flowy-error", features = ["impl_from_serde", "impl_from_dispatch_error", "impl_from_collab_document", "impl_from_collab_persistence"] } diff --git a/frontend/rust-lib/flowy-document/src/entities.rs b/frontend/rust-lib/flowy-document/src/entities.rs index ad5912dfeb..079d07cb0a 100644 --- a/frontend/rust-lib/flowy-document/src/entities.rs +++ b/frontend/rust-lib/flowy-document/src/entities.rs @@ -77,11 +77,12 @@ pub struct UploadFileParamsPB { pub workspace_id: String, #[pb(index = 2)] - #[validate(custom = "required_valid_path")] - pub local_file_path: String, + #[validate(custom = "required_not_empty_str")] + pub document_id: String, #[pb(index = 3)] - pub is_async: bool, + #[validate(custom = "required_valid_path")] + pub local_file_path: String, } #[derive(Default, ProtoBuf, Validate)] diff --git a/frontend/rust-lib/flowy-document/src/event_handler.rs b/frontend/rust-lib/flowy-document/src/event_handler.rs index 029405c6cc..0dca0d2ccb 100644 --- a/frontend/rust-lib/flowy-document/src/event_handler.rs +++ b/frontend/rust-lib/flowy-document/src/event_handler.rs @@ -422,13 +422,13 @@ pub(crate) async fn upload_file_handler( ) -> DataResult { let AFPluginData(UploadFileParamsPB { workspace_id, + document_id, local_file_path, - is_async, }) = params; let manager = upgrade_document(manager)?; let url = manager - .upload_file(workspace_id, &local_file_path, is_async) + .upload_file(workspace_id, &document_id, &local_file_path) .await?; Ok(AFPluginData(UploadedFilePB { diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs index 9edaccc796..5511a61725 100644 --- a/frontend/rust-lib/flowy-document/src/manager.rs +++ b/frontend/rust-lib/flowy-document/src/manager.rs @@ -13,16 +13,14 @@ use collab_document::document_data::default_document_data; use collab_entity::CollabType; use collab_plugins::CollabKVDB; use dashmap::DashMap; -use flowy_storage::object_from_disk; use lib_infra::util::timestamp; -use tokio::io::AsyncWriteExt; -use tracing::{error, trace}; +use tracing::trace; use tracing::{event, instrument}; use collab_integrate::collab_builder::{AppFlowyCollabBuilder, CollabBuilderConfig}; use flowy_document_pub::cloud::DocumentCloudService; use flowy_error::{internal_error, ErrorCode, FlowyError, FlowyResult}; -use flowy_storage::ObjectStorageService; +use flowy_storage_pub::storage::StorageService; use lib_dispatch::prelude::af_spawn; use crate::document::MutexDocument; @@ -53,7 +51,7 @@ pub struct DocumentManager { documents: Arc>>, removing_documents: Arc>>, cloud_service: Arc, - storage_service: Weak, + storage_service: Weak, snapshot_service: Arc, } @@ -62,7 +60,7 @@ impl DocumentManager { user_service: Arc, collab_builder: Arc, cloud_service: Arc, - storage_service: Weak, + storage_service: Weak, snapshot_service: Arc, ) -> Self { Self { @@ -323,73 +321,30 @@ impl DocumentManager { Ok(snapshot) } + #[instrument(level = "debug", skip_all, err)] pub async fn upload_file( &self, workspace_id: String, + document_id: &str, local_file_path: &str, - is_async: bool, ) -> FlowyResult { - let (object_identity, object_value) = object_from_disk(&workspace_id, local_file_path).await?; let storage_service = self.storage_service_upgrade()?; - let url = storage_service.get_object_url(object_identity).await?; - - let clone_url = url.clone(); - - match is_async { - false => storage_service.put_object(clone_url, object_value).await?, - true => { - // let the upload happen in the background - af_spawn(async move { - if let Err(e) = storage_service.put_object(clone_url, object_value).await { - error!("upload file failed: {}", e); - } - }); - }, - } + let url = storage_service + .create_upload(&workspace_id, document_id, local_file_path) + .await? + .url; Ok(url) } pub async fn download_file(&self, local_file_path: String, url: String) -> FlowyResult<()> { - // TODO(nathan): save file when the current target is wasm - #[cfg(not(target_arch = "wasm32"))] - { - if tokio::fs::metadata(&local_file_path).await.is_ok() { - tracing::warn!("file already exist in user local disk: {}", local_file_path); - return Ok(()); - } - - let storage_service = self.storage_service_upgrade()?; - let object_value = storage_service.get_object(url).await?; - // create file if not exist - let mut file = tokio::fs::OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&local_file_path) - .await?; - - let n = file.write(&object_value.raw).await?; - tracing::info!("downloaded {} bytes to file: {}", n, local_file_path); - } + let storage_service = self.storage_service_upgrade()?; + storage_service.download_object(url, local_file_path)?; Ok(()) } pub async fn delete_file(&self, local_file_path: String, url: String) -> FlowyResult<()> { - // TODO(nathan): delete file when the current target is wasm - #[cfg(not(target_arch = "wasm32"))] - // delete file from local - tokio::fs::remove_file(local_file_path).await?; - - // delete from cloud let storage_service = self.storage_service_upgrade()?; - af_spawn(async move { - if let Err(e) = storage_service.delete_object(url).await { - // TODO: add WAL to log the delete operation. - // keep a list of files to be deleted, and retry later - error!("delete file failed: {}", e); - } - }); - + storage_service.delete_object(url, local_file_path)?; Ok(()) } @@ -424,7 +379,7 @@ impl DocumentManager { } } - fn storage_service_upgrade(&self) -> FlowyResult> { + fn storage_service_upgrade(&self) -> FlowyResult> { let storage_service = self.storage_service.upgrade().ok_or_else(|| { FlowyError::internal().with_context("The file storage service is already dropped") })?; @@ -438,7 +393,7 @@ impl DocumentManager { } /// Only expose this method for testing #[cfg(debug_assertions)] - pub fn get_file_storage_service(&self) -> &Weak { + pub fn get_file_storage_service(&self) -> &Weak { &self.storage_service } diff --git a/frontend/rust-lib/flowy-document/tests/document/util.rs b/frontend/rust-lib/flowy-document/tests/document/util.rs index 12ad16f44c..075b8e62a8 100644 --- a/frontend/rust-lib/flowy-document/tests/document/util.rs +++ b/frontend/rust-lib/flowy-document/tests/document/util.rs @@ -20,8 +20,10 @@ use flowy_document::entities::{DocumentSnapshotData, DocumentSnapshotMeta}; use flowy_document::manager::{DocumentManager, DocumentSnapshotService, DocumentUserService}; use flowy_document_pub::cloud::*; use flowy_error::{ErrorCode, FlowyError, FlowyResult}; -use flowy_storage::ObjectStorageService; +use flowy_storage_pub::chunked_byte::ChunkedBytes; +use flowy_storage_pub::storage::{CreatedUpload, StorageService}; use lib_infra::async_trait::async_trait; +use lib_infra::box_any::BoxAny; use lib_infra::future::FutureResult; pub struct DocumentTest { @@ -32,7 +34,7 @@ impl DocumentTest { pub fn new() -> Self { let user = FakeUser::new(); let cloud_service = Arc::new(LocalTestDocumentCloudServiceImpl()); - let file_storage = Arc::new(DocumentTestFileStorageService) as Arc; + let file_storage = Arc::new(DocumentTestFileStorageService) as Arc; let document_snapshot = Arc::new(DocumentTestSnapshot); let builder = Arc::new(AppFlowyCollabBuilder::new( @@ -173,27 +175,44 @@ impl DocumentCloudService for LocalTestDocumentCloudServiceImpl { } pub struct DocumentTestFileStorageService; -impl ObjectStorageService for DocumentTestFileStorageService { - fn get_object_url( + +#[async_trait] +impl StorageService for DocumentTestFileStorageService { + fn upload_object( &self, - _object_id: flowy_storage::ObjectIdentity, + _workspace_id: &str, + _local_file_path: &str, ) -> FutureResult { todo!() } - fn put_object( + fn delete_object(&self, _url: String, _local_file_path: String) -> FlowyResult<()> { + todo!() + } + + fn download_object(&self, _url: String, _local_file_path: String) -> FlowyResult<()> { + todo!() + } + + fn create_upload( &self, - _url: String, - _object_value: flowy_storage::ObjectValue, - ) -> FutureResult<(), FlowyError> { + _workspace_id: &str, + _parent_dir: &str, + _local_file_path: &str, + ) -> FutureResult { todo!() } - fn delete_object(&self, _url: String) -> FutureResult<(), FlowyError> { + async fn start_upload(&self, _chunks: &ChunkedBytes, _record: &BoxAny) -> Result<(), FlowyError> { todo!() } - fn get_object(&self, _url: String) -> FutureResult { + async fn resume_upload( + &self, + _workspace_id: &str, + _parent_dir: &str, + _file_id: &str, + ) -> Result<(), FlowyError> { todo!() } } diff --git a/frontend/rust-lib/flowy-server/Cargo.toml b/frontend/rust-lib/flowy-server/Cargo.toml index 184731f204..dabcef78e7 100644 --- a/frontend/rust-lib/flowy-server/Cargo.toml +++ b/frontend/rust-lib/flowy-server/Cargo.toml @@ -42,6 +42,7 @@ flowy-server-pub = { workspace = true } flowy-search-pub = { workspace = true } flowy-encrypt = { workspace = true } flowy-storage = { workspace = true } +flowy-storage-pub = { workspace = true } flowy-chat-pub = { workspace = true } mime_guess = "2.0" url = "2.4" diff --git a/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs b/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs index 839a8b5ed1..868559bcbb 100644 --- a/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs +++ b/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs @@ -1,5 +1,8 @@ -use flowy_error::FlowyError; -use flowy_storage::{ObjectIdentity, ObjectStorageService, ObjectValue}; +use client_api::entity::{CompleteUploadRequest, CreateUploadRequest}; +use flowy_error::{FlowyError, FlowyResult}; +use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService}; +use flowy_storage_pub::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse}; +use lib_infra::async_trait::async_trait; use lib_infra::future::FutureResult; use crate::af_cloud::AFServer; @@ -12,7 +15,8 @@ impl AFCloudFileStorageServiceImpl { } } -impl ObjectStorageService for AFCloudFileStorageServiceImpl +#[async_trait] +impl StorageCloudService for AFCloudFileStorageServiceImpl where T: AFServer, { @@ -36,7 +40,8 @@ where }) } - fn delete_object(&self, url: String) -> FutureResult<(), FlowyError> { + fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError> { + let url = url.to_string(); let try_get_client = self.0.try_get_client(); FutureResult::new(async move { let client = try_get_client?; @@ -56,4 +61,84 @@ where }) }) } + + fn get_object_url_v1( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + ) -> FlowyResult { + let client = self.0.try_get_client()?; + let url = client.get_blob_url_v1(workspace_id, parent_dir, file_id); + Ok(url) + } + + async fn create_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + content_type: &str, + ) -> Result { + let parent_dir = parent_dir.to_string(); + let content_type = content_type.to_string(); + let file_id = file_id.to_string(); + let try_get_client = self.0.try_get_client(); + let client = try_get_client?; + let req = CreateUploadRequest { + file_id, + parent_dir, + content_type, + }; + let resp = client.create_upload(workspace_id, req).await?; + Ok(resp) + } + + async fn upload_part( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + part_number: i32, + body: Vec, + ) -> Result { + let try_get_client = self.0.try_get_client(); + let client = try_get_client?; + let resp = client + .upload_part( + workspace_id, + parent_dir, + file_id, + upload_id, + part_number, + body, + ) + .await?; + + Ok(resp) + } + + async fn complete_upload( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + parts: Vec, + ) -> Result<(), FlowyError> { + let parent_dir = parent_dir.to_string(); + let upload_id = upload_id.to_string(); + let file_id = file_id.to_string(); + let try_get_client = self.0.try_get_client(); + let client = try_get_client?; + let request = CompleteUploadRequest { + file_id, + parent_dir, + upload_id, + parts, + }; + client.complete_upload(workspace_id, request).await?; + Ok(()) + } } diff --git a/frontend/rust-lib/flowy-server/src/af_cloud/server.rs b/frontend/rust-lib/flowy-server/src/af_cloud/server.rs index 6e50eb3b59..c679883b69 100644 --- a/frontend/rust-lib/flowy-server/src/af_cloud/server.rs +++ b/frontend/rust-lib/flowy-server/src/af_cloud/server.rs @@ -12,7 +12,6 @@ use client_api::ws::{ use client_api::{Client, ClientConfiguration}; use flowy_chat_pub::cloud::ChatCloudService; use flowy_search_pub::cloud::SearchCloudService; -use flowy_storage::ObjectStorageService; use rand::Rng; use semver::Version; use tokio::select; @@ -28,6 +27,7 @@ use flowy_document_pub::cloud::DocumentCloudService; use flowy_error::{ErrorCode, FlowyError}; use flowy_folder_pub::cloud::FolderCloudService; use flowy_server_pub::af_cloud_config::AFCloudConfiguration; +use flowy_storage_pub::cloud::StorageCloudService; use flowy_user_pub::cloud::{UserCloudService, UserUpdate}; use flowy_user_pub::entities::UserTokenState; use lib_dispatch::prelude::af_spawn; @@ -252,7 +252,7 @@ impl AppFlowyServer for AppFlowyCloudServer { Ok(channel.map(|c| (c, connect_state_recv, self.ws_client.is_connected()))) } - fn file_storage(&self) -> Option> { + fn file_storage(&self) -> Option> { let client = AFServerImpl { client: self.get_client(), }; diff --git a/frontend/rust-lib/flowy-server/src/local_server/server.rs b/frontend/rust-lib/flowy-server/src/local_server/server.rs index b2c17c900d..5bb00d9c60 100644 --- a/frontend/rust-lib/flowy-server/src/local_server/server.rs +++ b/frontend/rust-lib/flowy-server/src/local_server/server.rs @@ -1,5 +1,4 @@ use flowy_search_pub::cloud::SearchCloudService; -use flowy_storage::ObjectStorageService; use std::sync::Arc; use parking_lot::RwLock; @@ -9,6 +8,7 @@ use flowy_database_pub::cloud::DatabaseCloudService; use flowy_document_pub::cloud::DocumentCloudService; use flowy_error::FlowyError; use flowy_folder_pub::cloud::FolderCloudService; +use flowy_storage_pub::cloud::StorageCloudService; // use flowy_user::services::database::{ // get_user_profile, get_user_workspace, open_collab_db, open_user_db, // }; @@ -68,7 +68,7 @@ impl AppFlowyServer for LocalServer { Arc::new(LocalServerDocumentCloudServiceImpl()) } - fn file_storage(&self) -> Option> { + fn file_storage(&self) -> Option> { None } diff --git a/frontend/rust-lib/flowy-server/src/server.rs b/frontend/rust-lib/flowy-server/src/server.rs index bebfc81bcb..d166e0632f 100644 --- a/frontend/rust-lib/flowy-server/src/server.rs +++ b/frontend/rust-lib/flowy-server/src/server.rs @@ -2,7 +2,6 @@ use client_api::ws::ConnectState; use client_api::ws::WSConnectStateReceiver; use client_api::ws::WebSocketChannel; use flowy_search_pub::cloud::SearchCloudService; -use flowy_storage::ObjectStorageService; use std::sync::Arc; use anyhow::Error; @@ -17,6 +16,7 @@ use crate::default_impl::DefaultChatCloudServiceImpl; use flowy_database_pub::cloud::DatabaseCloudService; use flowy_document_pub::cloud::DocumentCloudService; use flowy_folder_pub::cloud::FolderCloudService; +use flowy_storage_pub::cloud::StorageCloudService; use flowy_user_pub::cloud::UserCloudService; use flowy_user_pub::entities::UserTokenState; @@ -144,7 +144,7 @@ pub trait AppFlowyServer: Send + Sync + 'static { Ok(None) } - fn file_storage(&self) -> Option>; + fn file_storage(&self) -> Option>; } pub struct EncryptionImpl { diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs index 89dfc39971..6db01f1cc6 100644 --- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs +++ b/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs @@ -1,7 +1,5 @@ -use std::borrow::Cow; - use anyhow::Error; -use flowy_storage::StorageObject; +use flowy_storage_pub::cloud::StorageObject; use hyper::header::CONTENT_TYPE; use reqwest::header::IntoHeaderName; use reqwest::multipart::{Form, Part}; @@ -9,12 +7,14 @@ use reqwest::{ header::{HeaderMap, HeaderValue}, Client, Method, RequestBuilder, }; +use std::borrow::Cow; use tokio::fs::File; use tokio::io::AsyncReadExt; use url::Url; use crate::supabase::file_storage::{DeleteObjects, FileOptions, NewBucket, RequestBody}; +#[allow(dead_code)] pub struct StorageRequestBuilder { pub url: Url, headers: HeaderMap, @@ -23,6 +23,7 @@ pub struct StorageRequestBuilder { body: RequestBody, } +#[allow(dead_code)] impl StorageRequestBuilder { pub fn new(url: Url, headers: HeaderMap, client: Client) -> Self { Self { diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs index b00bf8f9a6..f150084c2d 100644 --- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs +++ b/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs @@ -1,141 +1,13 @@ -use std::sync::{Arc, Weak}; +#![allow(clippy::all)] +#![allow(unknown_lints)] +#![allow(unused_attributes)] +use std::sync::Weak; use anyhow::{anyhow, Error}; -use reqwest::{ - header::{HeaderMap, HeaderValue}, - Client, -}; use url::Url; -use flowy_encrypt::{decrypt_data, encrypt_data}; -use flowy_error::FlowyError; -use flowy_server_pub::supabase_config::SupabaseConfiguration; -use flowy_storage::{FileStoragePlan, ObjectStorageService}; -use lib_infra::future::FutureResult; - -use crate::supabase::file_storage::builder::StorageRequestBuilder; use crate::AppFlowyEncryption; - -pub struct SupabaseFileStorage { - url: Url, - headers: HeaderMap, - client: Client, - #[allow(dead_code)] - encryption: ObjectEncryption, - #[allow(dead_code)] - storage_plan: Arc, -} - -impl ObjectStorageService for SupabaseFileStorage { - fn get_object_url( - &self, - _object_id: flowy_storage::ObjectIdentity, - ) -> FutureResult { - todo!() - } - - fn put_object( - &self, - _url: String, - _object_value: flowy_storage::ObjectValue, - ) -> FutureResult<(), FlowyError> { - todo!() - } - - fn delete_object(&self, _url: String) -> FutureResult<(), FlowyError> { - todo!() - } - - fn get_object(&self, _url: String) -> FutureResult { - todo!() - } - - // fn create_object(&self, object: StorageObject) -> FutureResult { - // let mut storage = self.storage(); - // let storage_plan = Arc::downgrade(&self.storage_plan); - - // FutureResult::new(async move { - // let plan = storage_plan - // .upgrade() - // .ok_or(anyhow!("Storage plan is not available"))?; - // plan.check_upload_object(&object).await?; - - // storage = storage.upload_object("data", object); - // let url = storage.url.to_string(); - // storage.build().await?.send().await?.success().await?; - // Ok(url) - // }) - // } - - // fn delete_object_by_url(&self, object_url: String) -> FutureResult<(), FlowyError> { - // let storage = self.storage(); - - // FutureResult::new(async move { - // let url = Url::parse(&object_url)?; - // let location = get_object_location_from(&url)?; - // storage - // .delete_object(location.bucket_id, location.file_name) - // .build() - // .await? - // .send() - // .await? - // .success() - // .await?; - // Ok(()) - // }) - // } - - // fn get_object_by_url(&self, object_url: String) -> FutureResult { - // let storage = self.storage(); - // FutureResult::new(async move { - // let url = Url::parse(&object_url)?; - // let location = get_object_location_from(&url)?; - // let bytes = storage - // .get_object(location.bucket_id, location.file_name) - // .build() - // .await? - // .send() - // .await? - // .get_bytes() - // .await?; - // Ok(bytes) - // }) - // } -} - -impl SupabaseFileStorage { - pub fn new( - config: &SupabaseConfiguration, - encryption: Weak, - storage_plan: Arc, - ) -> Result { - let mut headers = HeaderMap::new(); - let url = format!("{}/storage/v1", config.url); - let auth = format!("Bearer {}", config.anon_key); - - headers.insert( - "Authorization", - HeaderValue::from_str(&auth).expect("Authorization is invalid"), - ); - headers.insert( - "apikey", - HeaderValue::from_str(&config.anon_key).expect("apikey value is invalid"), - ); - - let encryption = ObjectEncryption::new(encryption); - Ok(Self { - url: Url::parse(&url)?, - headers, - client: Client::new(), - encryption, - storage_plan, - }) - } - - pub fn storage(&self) -> StorageRequestBuilder { - StorageRequestBuilder::new(self.url.clone(), self.headers.clone(), self.client.clone()) - } -} +use flowy_encrypt::{decrypt_data, encrypt_data}; #[allow(dead_code)] struct ObjectEncryption { @@ -143,6 +15,7 @@ struct ObjectEncryption { } impl ObjectEncryption { + #[allow(dead_code)] fn new(encryption: Weak) -> Self { Self { encryption } } diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs index 768ae27b3e..ec1ffa6c7b 100644 --- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs +++ b/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs @@ -1,8 +1,7 @@ use bytes::Bytes; +use flowy_storage_pub::cloud::ObjectValueSupabase; use serde::{Deserialize, Serialize}; -use flowy_storage::ObjectValueSupabase; - use crate::supabase; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs index 63cf2cb6e0..01482b1099 100644 --- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs +++ b/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs @@ -3,7 +3,7 @@ use std::sync::Weak; use parking_lot::RwLock; use flowy_error::FlowyError; -use flowy_storage::{FileStoragePlan, StorageObject}; +use flowy_storage_pub::cloud::{FileStoragePlan, StorageObject}; use lib_infra::future::FutureResult; use crate::supabase::api::RESTfulPostgresServer; diff --git a/frontend/rust-lib/flowy-server/src/supabase/server.rs b/frontend/rust-lib/flowy-server/src/supabase/server.rs index fa59931fa2..7b88a80545 100644 --- a/frontend/rust-lib/flowy-server/src/supabase/server.rs +++ b/frontend/rust-lib/flowy-server/src/supabase/server.rs @@ -1,5 +1,4 @@ use flowy_search_pub::cloud::SearchCloudService; -use flowy_storage::ObjectStorageService; use std::collections::HashMap; use std::sync::{Arc, Weak}; @@ -11,6 +10,7 @@ use flowy_database_pub::cloud::DatabaseCloudService; use flowy_document_pub::cloud::DocumentCloudService; use flowy_folder_pub::cloud::FolderCloudService; use flowy_server_pub::supabase_config::SupabaseConfiguration; +use flowy_storage_pub::cloud::StorageCloudService; use flowy_user_pub::cloud::UserCloudService; use crate::supabase::api::{ @@ -18,8 +18,7 @@ use crate::supabase::api::{ SupabaseCollabStorageImpl, SupabaseDatabaseServiceImpl, SupabaseDocumentServiceImpl, SupabaseFolderServiceImpl, SupabaseServerServiceImpl, SupabaseUserServiceImpl, }; -use crate::supabase::file_storage::core::SupabaseFileStorage; -use crate::supabase::file_storage::FileStoragePlanImpl; + use crate::{AppFlowyEncryption, AppFlowyServer}; /// https://www.pgbouncer.org/features.html @@ -63,10 +62,10 @@ pub struct SupabaseServer { #[allow(dead_code)] config: SupabaseConfiguration, device_id: String, + #[allow(dead_code)] uid: Arc>>, collab_update_sender: Arc, restful_postgres: Arc>>>, - file_storage: Arc>>>, encryption: Weak, } @@ -87,23 +86,11 @@ impl SupabaseServer { } else { None }; - let file_storage = if enable_sync { - let plan = FileStoragePlanImpl::new( - Arc::downgrade(&uid), - restful_postgres.as_ref().map(Arc::downgrade), - ); - Some(Arc::new( - SupabaseFileStorage::new(&config, encryption.clone(), Arc::new(plan)).unwrap(), - )) - } else { - None - }; Self { config, device_id, collab_update_sender, restful_postgres: Arc::new(RwLock::new(restful_postgres)), - file_storage: Arc::new(RwLock::new(file_storage)), encryption, uid, } @@ -119,19 +106,8 @@ impl AppFlowyServer for SupabaseServer { let postgres = RESTfulPostgresServer::new(self.config.clone(), self.encryption.clone()); *self.restful_postgres.write() = Some(Arc::new(postgres)); } - - if self.file_storage.read().is_none() { - let plan = FileStoragePlanImpl::new( - Arc::downgrade(&self.uid), - self.restful_postgres.read().as_ref().map(Arc::downgrade), - ); - let file_storage = - SupabaseFileStorage::new(&self.config, self.encryption.clone(), Arc::new(plan)).unwrap(); - *self.file_storage.write() = Some(Arc::new(file_storage)); - } } else { *self.restful_postgres.write() = None; - *self.file_storage.write() = None; } } @@ -188,12 +164,8 @@ impl AppFlowyServer for SupabaseServer { ))) } - fn file_storage(&self) -> Option> { - self - .file_storage - .read() - .clone() - .map(|s| s as Arc) + fn file_storage(&self) -> Option> { + None } fn search_service(&self) -> Option> { diff --git a/frontend/rust-lib/flowy-server/tests/main.rs b/frontend/rust-lib/flowy-server/tests/main.rs index cd827b9b9d..fb12ed51b3 100644 --- a/frontend/rust-lib/flowy-server/tests/main.rs +++ b/frontend/rust-lib/flowy-server/tests/main.rs @@ -5,7 +5,7 @@ use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; mod af_cloud_test; -mod supabase_test; +// mod supabase_test; pub fn setup_log() { static START: Once = Once::new(); diff --git a/frontend/rust-lib/flowy-server/tests/supabase_test/util.rs b/frontend/rust-lib/flowy-server/tests/supabase_test/util.rs index a0f3d1fbdc..7fba91fe9a 100644 --- a/frontend/rust-lib/flowy-server/tests/supabase_test/util.rs +++ b/frontend/rust-lib/flowy-server/tests/supabase_test/util.rs @@ -1,4 +1,3 @@ -use flowy_storage::ObjectStorageService; use std::collections::HashMap; use std::sync::Arc; @@ -16,10 +15,8 @@ use flowy_server::supabase::api::{ SupabaseFolderServiceImpl, SupabaseServerServiceImpl, SupabaseUserServiceImpl, }; use flowy_server::supabase::define::{USER_DEVICE_ID, USER_EMAIL, USER_UUID}; -use flowy_server::supabase::file_storage::core::SupabaseFileStorage; use flowy_server::{AppFlowyEncryption, EncryptionImpl}; use flowy_server_pub::supabase_config::SupabaseConfiguration; -use flowy_storage::{FileStoragePlan, StorageObject}; use flowy_user_pub::cloud::UserCloudService; use lib_infra::future::FutureResult; @@ -63,7 +60,7 @@ pub fn folder_service() -> Arc { } #[allow(dead_code)] -pub fn file_storage_service() -> Arc { +pub fn file_storage_service() -> Arc { let encryption_impl: Arc = Arc::new(EncryptionImpl::new(None)); let config = SupabaseConfiguration::from_env().unwrap(); Arc::new( @@ -163,19 +160,3 @@ pub fn third_party_sign_up_param(uuid: String) -> HashMap { } pub struct TestFileStoragePlan; - -impl FileStoragePlan for TestFileStoragePlan { - fn storage_size(&self) -> FutureResult { - // 1 GB - FutureResult::new(async { Ok(1024 * 1024 * 1024) }) - } - - fn maximum_file_size(&self) -> FutureResult { - // 5 MB - FutureResult::new(async { Ok(5 * 1024 * 1024) }) - } - - fn check_upload_object(&self, _object: &StorageObject) -> FutureResult<(), FlowyError> { - FutureResult::new(async { Ok(()) }) - } -} diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/down.sql b/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/down.sql new file mode 100644 index 0000000000..dddef11ffa --- /dev/null +++ b/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +drop table upload_file_table; diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/up.sql b/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/up.sql new file mode 100644 index 0000000000..13c7b01e9a --- /dev/null +++ b/frontend/rust-lib/flowy-sqlite/migrations/2024-06-16-131359_file_upload/up.sql @@ -0,0 +1,20 @@ +-- Your SQL goes here +CREATE TABLE upload_file_table ( + workspace_id TEXT NOT NULL, + file_id TEXT NOT NULL, + parent_dir TEXT NOT NULL, + local_file_path TEXT NOT NULL, + content_type TEXT NOT NULL, + chunk_size INTEGER NOT NULL, + num_chunk INTEGER NOT NULL, + upload_id TEXT NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + PRIMARY KEY (workspace_id, parent_dir, file_id) +); + +CREATE TABLE upload_file_part ( + upload_id TEXT NOT NULL, + e_tag TEXT NOT NULL, + part_num INTEGER NOT NULL, + PRIMARY KEY (upload_id, e_tag) +); \ No newline at end of file diff --git a/frontend/rust-lib/flowy-sqlite/src/schema.rs b/frontend/rust-lib/flowy-sqlite/src/schema.rs index 5dbbc78501..49fcc254d1 100644 --- a/frontend/rust-lib/flowy-sqlite/src/schema.rs +++ b/frontend/rust-lib/flowy-sqlite/src/schema.rs @@ -32,6 +32,28 @@ diesel::table! { } } +diesel::table! { + upload_file_part (upload_id, e_tag) { + upload_id -> Text, + e_tag -> Text, + part_num -> Integer, + } +} + +diesel::table! { + upload_file_table (workspace_id, file_id, parent_dir) { + workspace_id -> Text, + file_id -> Text, + parent_dir -> Text, + local_file_path -> Text, + content_type -> Text, + chunk_size -> Integer, + num_chunk -> Integer, + upload_id -> Text, + created_at -> BigInt, + } +} + diesel::table! { user_data_migration_records (id) { id -> Integer, @@ -80,11 +102,13 @@ diesel::table! { } diesel::allow_tables_to_appear_in_same_query!( - chat_message_table, - chat_table, - collab_snapshot, - user_data_migration_records, - user_table, - user_workspace_table, - workspace_members_table, + chat_message_table, + chat_table, + collab_snapshot, + upload_file_part, + upload_file_table, + user_data_migration_records, + user_table, + user_workspace_table, + workspace_members_table, ); diff --git a/frontend/rust-lib/flowy-storage-pub/Cargo.toml b/frontend/rust-lib/flowy-storage-pub/Cargo.toml new file mode 100644 index 0000000000..3dee8f9f5e --- /dev/null +++ b/frontend/rust-lib/flowy-storage-pub/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "flowy-storage-pub" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +lib-infra.workspace = true +serde_json.workspace = true +serde.workspace = true +async-trait.workspace = true +mime = "0.3.17" +flowy-error = { workspace = true, features = ["impl_from_reqwest"] } +bytes.workspace = true +mime_guess = "2.0.4" +client-api-entity = { workspace = true } +tokio = { workspace = true, features = ["sync", "io-util"] } +anyhow = "1.0.86" diff --git a/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs b/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs new file mode 100644 index 0000000000..d1210ec8b0 --- /dev/null +++ b/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs @@ -0,0 +1,239 @@ +use anyhow::anyhow; +use bytes::Bytes; +use std::fmt::Display; +use std::ops::Deref; +use std::path::Path; + +use tokio::io::AsyncReadExt; + +/// In Amazon S3, the minimum chunk size for multipart uploads is 5 MB,except for the last part, +/// which can be smaller.(https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html) +pub const MIN_CHUNK_SIZE: usize = 5 * 1024 * 1024; // Minimum Chunk Size 5 MB +pub struct ChunkedBytes { + pub data: Bytes, + pub chunk_size: i32, + pub offsets: Vec<(usize, usize)>, + pub current_offset: i32, +} + +impl Deref for ChunkedBytes { + type Target = Bytes; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl Display for ChunkedBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "ChunkedBytes: chunk_size: {}, offsets: {:?}, current_offset: {}", + self.chunk_size, self.offsets, self.current_offset + ) + } +} + +impl ChunkedBytes { + pub fn from_bytes_with_chunk_size(data: Bytes, chunk_size: i32) -> Result { + if chunk_size < MIN_CHUNK_SIZE as i32 { + return Err(anyhow!( + "Chunk size should be greater than or equal to {}", + MIN_CHUNK_SIZE + )); + } + + let offsets = split_into_chunks(&data, chunk_size as usize); + Ok(ChunkedBytes { + data, + offsets, + chunk_size, + current_offset: 0, + }) + } + + /// Used to create a `ChunkedBytes` from a `Bytes` object. The default chunk size is 5 MB. + pub fn from_bytes(data: Bytes) -> Self { + let chunk_size = MIN_CHUNK_SIZE as i32; + let offsets = split_into_chunks(&data, MIN_CHUNK_SIZE); + ChunkedBytes { + data, + offsets, + chunk_size, + current_offset: 0, + } + } + + pub async fn from_file>( + file_path: P, + chunk_size: i32, + ) -> Result { + let mut file = tokio::fs::File::open(file_path).await?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).await?; + let data = Bytes::from(buffer); + + let offsets = split_into_chunks(&data, chunk_size as usize); + Ok(ChunkedBytes { + data, + offsets, + chunk_size, + current_offset: 0, + }) + } + + pub fn set_current_offset(&mut self, offset: i32) { + self.current_offset = offset; + } + + pub fn iter(&self) -> ChunkedBytesIterator { + ChunkedBytesIterator { + chunked_data: self, + current_index: self.current_offset as usize, + } + } +} + +pub struct ChunkedBytesIterator<'a> { + chunked_data: &'a ChunkedBytes, + current_index: usize, +} +impl<'a> Iterator for ChunkedBytesIterator<'a> { + type Item = Bytes; + + fn next(&mut self) -> Option { + if self.current_index >= self.chunked_data.offsets.len() { + None + } else { + let (start, end) = self.chunked_data.offsets[self.current_index]; + self.current_index += 1; + Some(self.chunked_data.data.slice(start..end)) + } + } +} +// Function to split input bytes into several chunks and return offsets +pub fn split_into_chunks(data: &Bytes, chunk_size: usize) -> Vec<(usize, usize)> { + let mut offsets = Vec::new(); + let mut start = 0; + + while start < data.len() { + let end = std::cmp::min(start + chunk_size, data.len()); + offsets.push((start, end)); + start = end; + } + offsets +} + +// Function to get chunk data using chunk number +pub async fn get_chunk( + data: Bytes, + chunk_number: usize, + offsets: &[(usize, usize)], +) -> Result { + if chunk_number >= offsets.len() { + return Err(anyhow!("Chunk number out of range")); + } + + let (start, end) = offsets[chunk_number]; + let chunk = data.slice(start..end); + + Ok(chunk) +} + +#[cfg(test)] +mod tests { + use crate::chunked_byte::{ChunkedBytes, MIN_CHUNK_SIZE}; + use bytes::Bytes; + use std::env::temp_dir; + use tokio::io::AsyncWriteExt; + + #[tokio::test] + async fn test_chunked_bytes_less_than_chunk_size() { + let data = Bytes::from(vec![0; 1024 * 1024]); // 1 MB of zeroes + let chunked_data = + ChunkedBytes::from_bytes_with_chunk_size(data.clone(), MIN_CHUNK_SIZE as i32).unwrap(); + + // Check if the offsets are correct + assert_eq!(chunked_data.offsets.len(), 1); // Should have 1 chunk + assert_eq!(chunked_data.offsets[0], (0, 1024 * 1024)); + + // Check if the data can be iterated correctly + let mut iter = chunked_data.iter(); + assert_eq!(iter.next().unwrap().len(), 1024 * 1024); + assert!(iter.next().is_none()); + } + + #[tokio::test] + async fn test_chunked_bytes_from_bytes() { + let data = Bytes::from(vec![0; 15 * 1024 * 1024]); // 15 MB of zeroes + let chunked_data = + ChunkedBytes::from_bytes_with_chunk_size(data.clone(), MIN_CHUNK_SIZE as i32).unwrap(); + + // Check if the offsets are correct + assert_eq!(chunked_data.offsets.len(), 3); // Should have 3 chunks + assert_eq!(chunked_data.offsets[0], (0, 5 * 1024 * 1024)); + assert_eq!(chunked_data.offsets[1], (5 * 1024 * 1024, 10 * 1024 * 1024)); + assert_eq!( + chunked_data.offsets[2], + (10 * 1024 * 1024, 15 * 1024 * 1024) + ); + + // Check if the data can be iterated correctly + let mut iter = chunked_data.iter(); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert!(iter.next().is_none()); + } + + #[tokio::test] + async fn test_chunked_bytes_from_file() { + // Create a temporary file with 15 MB of zeroes + let mut file_path = temp_dir(); + file_path.push("test_file"); + + let mut file = tokio::fs::File::create(&file_path).await.unwrap(); + file.write_all(&vec![0; 15 * 1024 * 1024]).await.unwrap(); + file.flush().await.unwrap(); + + // Read the file into ChunkedBytes + let chunked_data = ChunkedBytes::from_file(&file_path, MIN_CHUNK_SIZE as i32) + .await + .unwrap(); + + // Check if the offsets are correct + assert_eq!(chunked_data.offsets.len(), 3); // Should have 3 chunks + assert_eq!(chunked_data.offsets[0], (0, 5 * 1024 * 1024)); + assert_eq!(chunked_data.offsets[1], (5 * 1024 * 1024, 10 * 1024 * 1024)); + assert_eq!( + chunked_data.offsets[2], + (10 * 1024 * 1024, 15 * 1024 * 1024) + ); + + // Check if the data can be iterated correctly + let mut iter = chunked_data.iter(); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); + assert!(iter.next().is_none()); + + // Clean up the temporary file + tokio::fs::remove_file(file_path).await.unwrap(); + } + + #[tokio::test] + async fn test_chunked_bytes_with_current_offset() { + let data = Bytes::from(vec![0; 15 * 1024 * 1024]); // 15 MB of zeroes + let mut chunked_data = + ChunkedBytes::from_bytes_with_chunk_size(data.clone(), MIN_CHUNK_SIZE as i32).unwrap(); + + // Set the current offset to the second chunk + chunked_data.set_current_offset(1); + + // Check if the iterator starts from the second chunk + let mut iter = chunked_data.iter(); + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); // Second chunk + assert_eq!(iter.next().unwrap().len(), 5 * 1024 * 1024); // Third chunk + assert!(iter.next().is_none()); + } +} diff --git a/frontend/rust-lib/flowy-storage-pub/src/cloud.rs b/frontend/rust-lib/flowy-storage-pub/src/cloud.rs new file mode 100644 index 0000000000..816de17beb --- /dev/null +++ b/frontend/rust-lib/flowy-storage-pub/src/cloud.rs @@ -0,0 +1,176 @@ +use crate::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse}; +use async_trait::async_trait; +use bytes::Bytes; +use flowy_error::{FlowyError, FlowyResult}; +use lib_infra::future::FutureResult; +use mime::Mime; + +#[async_trait] +pub trait StorageCloudService: Send + Sync { + /// Creates a new storage object. + /// + /// # Parameters + /// - `url`: url of the object to be created. + /// + /// # Returns + /// - `Ok()` + /// - `Err(Error)`: An error occurred during the operation. + fn get_object_url(&self, object_id: ObjectIdentity) -> FutureResult; + + /// Creates a new storage object. + /// + /// # Parameters + /// - `url`: url of the object to be created. + /// + /// # Returns + /// - `Ok()` + /// - `Err(Error)`: An error occurred during the operation. + fn put_object(&self, url: String, object_value: ObjectValue) -> FutureResult<(), FlowyError>; + + /// Deletes a storage object by its URL. + /// + /// # Parameters + /// - `url`: url of the object to be deleted. + /// + /// # Returns + /// - `Ok()` + /// - `Err(Error)`: An error occurred during the operation. + fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError>; + + /// Fetches a storage object by its URL. + /// + /// # Parameters + /// - `url`: url of the object + /// + /// # Returns + /// - `Ok(File)`: The returned file object. + /// - `Err(Error)`: An error occurred during the operation. + fn get_object(&self, url: String) -> FutureResult; + fn get_object_url_v1( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + ) -> FlowyResult; + + async fn create_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + content_type: &str, + ) -> Result; + + async fn upload_part( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + part_number: i32, + body: Vec, + ) -> Result; + + async fn complete_upload( + &self, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + parts: Vec, + ) -> Result<(), FlowyError>; +} + +pub trait FileStoragePlan: Send + Sync + 'static { + fn storage_size(&self) -> FutureResult; + fn maximum_file_size(&self) -> FutureResult; + + fn check_upload_object(&self, object: &StorageObject) -> FutureResult<(), FlowyError>; +} + +pub struct ObjectIdentity { + pub workspace_id: String, + pub file_id: String, + pub ext: String, +} + +#[derive(Clone)] +pub struct ObjectValue { + pub raw: Bytes, + pub mime: Mime, +} + +pub struct StorageObject { + pub workspace_id: String, + pub file_name: String, + pub value: ObjectValueSupabase, +} + +pub enum ObjectValueSupabase { + File { file_path: String }, + Bytes { bytes: Bytes, mime: String }, +} + +impl ObjectValueSupabase { + pub fn mime_type(&self) -> String { + match self { + ObjectValueSupabase::File { file_path } => mime_guess::from_path(file_path) + .first_or_octet_stream() + .to_string(), + ObjectValueSupabase::Bytes { mime, .. } => mime.clone(), + } + } +} + +impl StorageObject { + /// Creates a `StorageObject` from a file. + /// + /// # Parameters + /// + /// * `name`: The name of the storage object. + /// * `file_path`: The file path to the storage object's data. + /// + pub fn from_file(workspace_id: &str, file_name: &str, file_path: T) -> Self { + Self { + workspace_id: workspace_id.to_string(), + file_name: file_name.to_string(), + value: ObjectValueSupabase::File { + file_path: file_path.to_string(), + }, + } + } + + /// Creates a `StorageObject` from bytes. + /// + /// # Parameters + /// + /// * `name`: The name of the storage object. + /// * `bytes`: The byte data of the storage object. + /// * `mime`: The MIME type of the storage object. + /// + pub fn from_bytes>( + workspace_id: &str, + file_name: &str, + bytes: B, + mime: String, + ) -> Self { + let bytes = bytes.into(); + Self { + workspace_id: workspace_id.to_string(), + file_name: file_name.to_string(), + value: ObjectValueSupabase::Bytes { bytes, mime }, + } + } + + /// Gets the file size of the `StorageObject`. + /// + /// # Returns + /// + /// The file size in bytes. + pub fn file_size(&self) -> u64 { + match &self.value { + ObjectValueSupabase::File { file_path } => std::fs::metadata(file_path).unwrap().len(), + ObjectValueSupabase::Bytes { bytes, .. } => bytes.len() as u64, + } + } +} diff --git a/frontend/rust-lib/flowy-storage-pub/src/lib.rs b/frontend/rust-lib/flowy-storage-pub/src/lib.rs new file mode 100644 index 0000000000..fa646847a8 --- /dev/null +++ b/frontend/rust-lib/flowy-storage-pub/src/lib.rs @@ -0,0 +1,3 @@ +pub mod chunked_byte; +pub mod cloud; +pub mod storage; diff --git a/frontend/rust-lib/flowy-storage-pub/src/storage.rs b/frontend/rust-lib/flowy-storage-pub/src/storage.rs new file mode 100644 index 0000000000..8d40bef64c --- /dev/null +++ b/frontend/rust-lib/flowy-storage-pub/src/storage.rs @@ -0,0 +1,53 @@ +use crate::chunked_byte::ChunkedBytes; +use async_trait::async_trait; +pub use client_api_entity::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse}; +use flowy_error::{FlowyError, FlowyResult}; +use lib_infra::box_any::BoxAny; +use lib_infra::future::FutureResult; + +#[async_trait] +pub trait StorageService: Send + Sync { + fn upload_object( + &self, + workspace_id: &str, + local_file_path: &str, + ) -> FutureResult; + + fn delete_object(&self, url: String, local_file_path: String) -> FlowyResult<()>; + + fn download_object(&self, url: String, local_file_path: String) -> FlowyResult<()>; + + fn create_upload( + &self, + workspace_id: &str, + parent_dir: &str, + local_file_path: &str, + ) -> FutureResult; + + async fn start_upload(&self, chunks: &ChunkedBytes, record: &BoxAny) -> Result<(), FlowyError>; + + async fn resume_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + ) -> Result<(), FlowyError>; +} + +pub struct CreatedUpload { + pub url: String, + pub file_id: String, +} + +#[derive(Debug, Clone)] +pub struct UploadResult { + pub file_id: String, + pub status: UploadStatus, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum UploadStatus { + Finish, + Failed, + InProgress, +} diff --git a/frontend/rust-lib/flowy-storage/Cargo.toml b/frontend/rust-lib/flowy-storage/Cargo.toml index d35c17565e..aa2fe73042 100644 --- a/frontend/rust-lib/flowy-storage/Cargo.toml +++ b/frontend/rust-lib/flowy-storage/Cargo.toml @@ -3,21 +3,24 @@ name = "flowy-storage" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -crate-type = ["cdylib", "rlib"] - [dependencies] -reqwest = { version = "0.11", features = ["json", "stream"] } +flowy-storage-pub.workspace = true serde_json.workspace = true serde.workspace = true async-trait.workspace = true bytes.workspace = true -mime_guess = "2.0" lib-infra = { workspace = true } url = "2.2.2" -flowy-error = { workspace = true, features = ["impl_from_reqwest"] } -mime = "0.3.17" -tokio = { workspace = true, features = ["sync", "io-util"]} +flowy-error = { workspace = true, features = ["impl_from_reqwest", "impl_from_sqlite"] } +tokio = { workspace = true, features = ["sync", "io-util"] } tracing.workspace = true -fxhash = "0.2.1" \ No newline at end of file +flowy-sqlite.workspace = true +mime_guess = "2.0.4" +fxhash = "0.2.1" +anyhow = "1.0.86" +chrono = "0.4.33" + +[dev-dependencies] +tokio = { workspace = true, features = ["full"] } +uuid = "1.6.1" +rand = { version = "0.8", features = ["std_rng"] } diff --git a/frontend/rust-lib/flowy-storage/src/file_cache.rs b/frontend/rust-lib/flowy-storage/src/file_cache.rs new file mode 100644 index 0000000000..106d2824ee --- /dev/null +++ b/frontend/rust-lib/flowy-storage/src/file_cache.rs @@ -0,0 +1,88 @@ +use std::path::{Path, PathBuf}; +use tokio::fs::{self, File}; +use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; +use tracing::error; + +/// [FileTempStorage] is used to store the temporary files for uploading. After the file is uploaded, +/// the file will be deleted. +pub struct FileTempStorage { + storage_dir: PathBuf, +} + +impl FileTempStorage { + /// Creates a new `FileTempStorage` with the specified temporary directory. + pub fn new(storage_dir: PathBuf) -> Self { + if !storage_dir.exists() { + if let Err(err) = std::fs::create_dir_all(&storage_dir) { + error!("Failed to create temporary storage directory: {:?}", err); + } + } + + FileTempStorage { storage_dir } + } + + /// Generates a temporary file path using the given file name. + fn generate_temp_file_path_with_name(&self, file_name: &str) -> PathBuf { + self.storage_dir.join(file_name) + } + + /// Creates a temporary file from an existing local file path. + pub async fn create_temp_file_from_existing( + &self, + existing_file_path: &Path, + ) -> io::Result { + let file_name = existing_file_path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid file name"))? + .to_str() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid file name"))?; + + let temp_file_path = self.generate_temp_file_path_with_name(file_name); + fs::copy(existing_file_path, &temp_file_path).await?; + Ok( + temp_file_path + .to_str() + .ok_or(io::Error::new( + io::ErrorKind::InvalidInput, + "Invalid file path", + ))? + .to_owned(), + ) + } + + /// Creates a temporary file from bytes and a specified file name. + #[allow(dead_code)] + pub async fn create_temp_file_from_bytes( + &self, + file_name: &str, + data: &[u8], + ) -> io::Result { + let temp_file_path = self.generate_temp_file_path_with_name(file_name); + let mut file = File::create(&temp_file_path).await?; + file.write_all(data).await?; + Ok(temp_file_path) + } + + /// Writes data to the specified temporary file. + #[allow(dead_code)] + pub async fn write_to_temp_file(&self, file_path: &Path, data: &[u8]) -> io::Result<()> { + let mut file = File::create(file_path).await?; + file.write_all(data).await?; + Ok(()) + } + + /// Reads data from the specified temporary file. + #[allow(dead_code)] + pub async fn read_from_temp_file(&self, file_path: &Path) -> io::Result> { + let mut file = File::open(file_path).await?; + let mut data = Vec::new(); + file.read_to_end(&mut data).await?; + Ok(data) + } + + /// Deletes the specified temporary file. + pub async fn delete_temp_file>(&self, file_path: T) -> io::Result<()> { + fs::remove_file(file_path).await?; + Ok(()) + } +} diff --git a/frontend/rust-lib/flowy-storage/src/lib.rs b/frontend/rust-lib/flowy-storage/src/lib.rs index b318b55cb5..45e528233e 100644 --- a/frontend/rust-lib/flowy-storage/src/lib.rs +++ b/frontend/rust-lib/flowy-storage/src/lib.rs @@ -1,154 +1,4 @@ -if_native! { - mod native; - pub use native::*; -} - -if_wasm! { - mod wasm; - pub use wasm::*; -} - -use bytes::Bytes; - -use flowy_error::FlowyError; -use lib_infra::future::FutureResult; -use lib_infra::{conditional_send_sync_trait, if_native, if_wasm}; -use mime::Mime; - -pub struct ObjectIdentity { - pub workspace_id: String, - pub file_id: String, - pub ext: String, -} - -#[derive(Clone)] -pub struct ObjectValue { - pub raw: Bytes, - pub mime: Mime, -} -conditional_send_sync_trait! { - "Provides a service for object storage. The trait includes methods for CRUD operations on storage objects."; - ObjectStorageService { - /// Creates a new storage object. - /// - /// # Parameters - /// - `url`: url of the object to be created. - /// - /// # Returns - /// - `Ok()` - /// - `Err(Error)`: An error occurred during the operation. - fn get_object_url(&self, object_id: ObjectIdentity) -> FutureResult; - - /// Creates a new storage object. - /// - /// # Parameters - /// - `url`: url of the object to be created. - /// - /// # Returns - /// - `Ok()` - /// - `Err(Error)`: An error occurred during the operation. - fn put_object(&self, url: String, object_value: ObjectValue) -> FutureResult<(), FlowyError>; - - /// Deletes a storage object by its URL. - /// - /// # Parameters - /// - `url`: url of the object to be deleted. - /// - /// # Returns - /// - `Ok()` - /// - `Err(Error)`: An error occurred during the operation. - fn delete_object(&self, url: String) -> FutureResult<(), FlowyError>; - - /// Fetches a storage object by its URL. - /// - /// # Parameters - /// - `url`: url of the object - /// - /// # Returns - /// - `Ok(File)`: The returned file object. - /// - `Err(Error)`: An error occurred during the operation. - fn get_object(&self, url: String) -> FutureResult; - } -} - -pub trait FileStoragePlan: Send + Sync + 'static { - fn storage_size(&self) -> FutureResult; - fn maximum_file_size(&self) -> FutureResult; - - fn check_upload_object(&self, object: &StorageObject) -> FutureResult<(), FlowyError>; -} - -pub struct StorageObject { - pub workspace_id: String, - pub file_name: String, - pub value: ObjectValueSupabase, -} - -pub enum ObjectValueSupabase { - File { file_path: String }, - Bytes { bytes: Bytes, mime: String }, -} - -impl ObjectValueSupabase { - pub fn mime_type(&self) -> String { - match self { - ObjectValueSupabase::File { file_path } => mime_guess::from_path(file_path) - .first_or_octet_stream() - .to_string(), - ObjectValueSupabase::Bytes { mime, .. } => mime.clone(), - } - } -} - -impl StorageObject { - /// Creates a `StorageObject` from a file. - /// - /// # Parameters - /// - /// * `name`: The name of the storage object. - /// * `file_path`: The file path to the storage object's data. - /// - pub fn from_file(workspace_id: &str, file_name: &str, file_path: T) -> Self { - Self { - workspace_id: workspace_id.to_string(), - file_name: file_name.to_string(), - value: ObjectValueSupabase::File { - file_path: file_path.to_string(), - }, - } - } - - /// Creates a `StorageObject` from bytes. - /// - /// # Parameters - /// - /// * `name`: The name of the storage object. - /// * `bytes`: The byte data of the storage object. - /// * `mime`: The MIME type of the storage object. - /// - pub fn from_bytes>( - workspace_id: &str, - file_name: &str, - bytes: B, - mime: String, - ) -> Self { - let bytes = bytes.into(); - Self { - workspace_id: workspace_id.to_string(), - file_name: file_name.to_string(), - value: ObjectValueSupabase::Bytes { bytes, mime }, - } - } - - /// Gets the file size of the `StorageObject`. - /// - /// # Returns - /// - /// The file size in bytes. - pub fn file_size(&self) -> u64 { - match &self.value { - ObjectValueSupabase::File { file_path } => std::fs::metadata(file_path).unwrap().len(), - ObjectValueSupabase::Bytes { bytes, .. } => bytes.len() as u64, - } - } -} +mod file_cache; +pub mod manager; +pub mod sqlite_sql; +mod uploader; diff --git a/frontend/rust-lib/flowy-storage/src/manager.rs b/frontend/rust-lib/flowy-storage/src/manager.rs new file mode 100644 index 0000000000..8375b12f66 --- /dev/null +++ b/frontend/rust-lib/flowy-storage/src/manager.rs @@ -0,0 +1,648 @@ +use crate::file_cache::FileTempStorage; +use crate::sqlite_sql::{ + batch_select_upload_file, delete_upload_file, insert_upload_file, insert_upload_part, + select_upload_file, select_upload_parts, update_upload_file_upload_id, UploadFilePartTable, + UploadFileTable, +}; +use crate::uploader::{FileUploader, FileUploaderRunner, Signal, UploadTask, UploadTaskQueue}; +use async_trait::async_trait; +use flowy_error::{FlowyError, FlowyResult}; +use flowy_sqlite::DBConnection; +use flowy_storage_pub::chunked_byte::{ChunkedBytes, MIN_CHUNK_SIZE}; +use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService}; +use flowy_storage_pub::storage::{ + CompletedPartRequest, CreatedUpload, StorageService, UploadPartResponse, UploadResult, + UploadStatus, +}; +use lib_infra::box_any::BoxAny; +use lib_infra::future::FutureResult; +use lib_infra::util::timestamp; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::watch; +use tracing::{debug, error, info, instrument, trace}; + +pub trait StorageUserService: Send + Sync + 'static { + fn user_id(&self) -> Result; + fn workspace_id(&self) -> Result; + fn sqlite_connection(&self, uid: i64) -> Result; + fn get_application_root_dir(&self) -> &str; +} + +pub struct StorageManager { + pub storage_service: Arc, + uploader: Arc, + broadcast: tokio::sync::broadcast::Sender, +} + +impl Drop for StorageManager { + fn drop(&mut self) { + info!("[File] StorageManager is dropped"); + } +} + +impl StorageManager { + pub fn new( + cloud_service: Arc, + user_service: Arc, + ) -> Self { + let temp_storage_path = PathBuf::from(format!( + "{}/cache_files", + user_service.get_application_root_dir() + )); + let temp_storage = Arc::new(FileTempStorage::new(temp_storage_path)); + let (notifier, notifier_rx) = watch::channel(Signal::Proceed); + let (broadcast, _) = tokio::sync::broadcast::channel::(100); + let task_queue = Arc::new(UploadTaskQueue::new(notifier)); + let storage_service = Arc::new(StorageServiceImpl { + cloud_service, + user_service: user_service.clone(), + temp_storage, + task_queue: task_queue.clone(), + upload_status_notifier: broadcast.clone(), + }); + + let uploader = Arc::new(FileUploader::new(storage_service.clone(), task_queue)); + tokio::spawn(FileUploaderRunner::run( + Arc::downgrade(&uploader), + notifier_rx, + )); + + let weak_uploader = Arc::downgrade(&uploader); + tokio::spawn(async move { + // Start uploading after 30 seconds + tokio::time::sleep(Duration::from_secs(30)).await; + if let Some(uploader) = weak_uploader.upgrade() { + if let Err(err) = prepare_upload_task(uploader, user_service).await { + error!("prepare upload task failed: {}", err); + } + } + }); + + Self { + storage_service, + uploader, + broadcast, + } + } + + pub fn update_network_reachable(&self, reachable: bool) { + if reachable { + self.uploader.resume(); + } else { + self.uploader.pause(); + } + } + + pub fn subscribe_upload_result(&self) -> tokio::sync::broadcast::Receiver { + self.broadcast.subscribe() + } +} + +async fn prepare_upload_task( + uploader: Arc, + user_service: Arc, +) -> FlowyResult<()> { + let uid = user_service.user_id()?; + let conn = user_service.sqlite_connection(uid)?; + let upload_files = batch_select_upload_file(conn, 100)?; + let tasks = upload_files + .into_iter() + .map(|upload_file| UploadTask::BackgroundTask { + workspace_id: upload_file.workspace_id, + file_id: upload_file.file_id, + parent_dir: upload_file.parent_dir, + created_at: upload_file.created_at, + retry_count: 0, + }) + .collect::>(); + info!("prepare upload task: {}", tasks.len()); + uploader.queue_tasks(tasks).await; + Ok(()) +} + +pub struct StorageServiceImpl { + cloud_service: Arc, + user_service: Arc, + temp_storage: Arc, + task_queue: Arc, + upload_status_notifier: tokio::sync::broadcast::Sender, +} + +#[async_trait] +impl StorageService for StorageServiceImpl { + fn upload_object( + &self, + workspace_id: &str, + local_file_path: &str, + ) -> FutureResult { + let cloud_service = self.cloud_service.clone(); + let workspace_id = workspace_id.to_string(); + let local_file_path = local_file_path.to_string(); + FutureResult::new(async move { + let (object_identity, object_value) = + object_from_disk(&workspace_id, &local_file_path).await?; + let url = cloud_service.get_object_url(object_identity).await?; + match cloud_service.put_object(url.clone(), object_value).await { + Ok(_) => { + debug!("[File] success uploaded file to cloud: {}", url); + }, + Err(err) => { + error!("[File] upload file failed: {}", err); + return Err(err); + }, + } + Ok(url) + }) + } + + fn delete_object(&self, url: String, local_file_path: String) -> FlowyResult<()> { + let cloud_service = self.cloud_service.clone(); + tokio::spawn(async move { + match tokio::fs::remove_file(&local_file_path).await { + Ok(_) => { + debug!("[File] deleted file from local disk: {}", local_file_path) + }, + Err(err) => { + error!("[File] delete file at {} failed: {}", local_file_path, err); + }, + } + if let Err(e) = cloud_service.delete_object(&url).await { + // TODO: add WAL to log the delete operation. + // keep a list of files to be deleted, and retry later + error!("[File] delete file failed: {}", e); + } + debug!("[File] deleted file from cloud: {}", url); + }); + Ok(()) + } + + fn download_object(&self, url: String, local_file_path: String) -> FlowyResult<()> { + let cloud_service = self.cloud_service.clone(); + tokio::spawn(async move { + if tokio::fs::metadata(&local_file_path).await.is_ok() { + tracing::warn!("file already exist in user local disk: {}", local_file_path); + return Ok(()); + } + let object_value = cloud_service.get_object(url).await?; + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&local_file_path) + .await?; + + match file.write(&object_value.raw).await { + Ok(n) => { + info!("downloaded {} bytes to file: {}", n, local_file_path); + }, + Err(err) => { + error!("write file failed: {}", err); + }, + } + Ok::<_, FlowyError>(()) + }); + Ok(()) + } + + fn create_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_path: &str, + ) -> FutureResult { + if workspace_id.is_empty() { + return FutureResult::new(async { + Err(FlowyError::internal().with_context("workspace id is empty")) + }); + } + + if parent_dir.is_empty() { + return FutureResult::new(async { + Err(FlowyError::internal().with_context("parent dir is empty")) + }); + } + + if file_path.is_empty() { + return FutureResult::new(async { + Err(FlowyError::internal().with_context("local file path is empty")) + }); + } + + let workspace_id = workspace_id.to_string(); + let parent_dir = parent_dir.to_string(); + let file_path = file_path.to_string(); + let temp_storage = self.temp_storage.clone(); + let task_queue = self.task_queue.clone(); + let user_service = self.user_service.clone(); + let cloud_service = self.cloud_service.clone(); + + FutureResult::new(async move { + let local_file_path = temp_storage + .create_temp_file_from_existing(Path::new(&file_path)) + .await + .map_err(|err| { + error!("[File] create temp file failed: {}", err); + FlowyError::internal() + .with_context(format!("create temp file for upload file failed: {}", err)) + })?; + + // 1. create a file record and chunk the file + let (chunks, record) = + create_upload_record(workspace_id, parent_dir, local_file_path).await?; + + // 2. save the record to sqlite + let conn = user_service.sqlite_connection(user_service.user_id()?)?; + insert_upload_file(conn, &record)?; + + // 3. generate url for given file + let url = cloud_service.get_object_url_v1( + &record.workspace_id, + &record.parent_dir, + &record.file_id, + )?; + let file_id = record.file_id.clone(); + + task_queue + .queue_task(UploadTask::Task { + chunks, + record, + retry_count: 0, + }) + .await; + + Ok::<_, FlowyError>(CreatedUpload { url, file_id }) + }) + } + + async fn start_upload(&self, chunks: &ChunkedBytes, record: &BoxAny) -> Result<(), FlowyError> { + let file_record = record.downcast_ref::().ok_or_else(|| { + FlowyError::internal().with_context("failed to downcast record to UploadFileTable") + })?; + + if let Err(err) = start_upload( + &self.cloud_service, + &self.user_service, + &self.temp_storage, + chunks, + file_record, + self.upload_status_notifier.clone(), + ) + .await + { + error!("[File] start upload failed: {}", err); + } + Ok(()) + } + + async fn resume_upload( + &self, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + ) -> Result<(), FlowyError> { + // Gathering the upload record and parts from the sqlite database. + let record = { + let mut conn = self + .user_service + .sqlite_connection(self.user_service.user_id()?)?; + conn.immediate_transaction(|conn| { + Ok::<_, FlowyError>( + // When resuming an upload, check if the upload_id is empty. + // If the upload_id is empty, the upload has likely not been created yet. + // If the upload_id is not empty, verify which parts have already been uploaded. + select_upload_file(conn, &workspace_id, &parent_dir, &file_id)?.and_then(|record| { + if record.upload_id.is_empty() { + Some((record, vec![])) + } else { + let parts = select_upload_parts(conn, &record.upload_id).unwrap_or_default(); + Some((record, parts)) + } + }), + ) + })? + }; + + if let Some((upload_file, parts)) = record { + resume_upload( + &self.cloud_service, + &self.user_service, + &self.temp_storage, + upload_file, + parts, + self.upload_status_notifier.clone(), + ) + .await?; + } else { + error!("[File] resume upload failed: record not found"); + } + Ok(()) + } +} + +async fn create_upload_record( + workspace_id: String, + parent_dir: String, + local_file_path: String, +) -> FlowyResult<(ChunkedBytes, UploadFileTable)> { + // read file and chunk it base on CHUNK_SIZE. We use MIN_CHUNK_SIZE as the minimum chunk size + let chunked_bytes = ChunkedBytes::from_file(&local_file_path, MIN_CHUNK_SIZE as i32).await?; + let ext = Path::new(&local_file_path) + .extension() + .and_then(std::ffi::OsStr::to_str) + .unwrap_or("") + .to_owned(); + let content_type = mime_guess::from_path(&local_file_path) + .first_or_octet_stream() + .to_string(); + let file_id = format!("{}.{}", fxhash::hash(&chunked_bytes.data).to_string(), ext); + let record = UploadFileTable { + workspace_id, + file_id, + upload_id: "".to_string(), + parent_dir, + local_file_path, + content_type, + chunk_size: chunked_bytes.chunk_size, + num_chunk: chunked_bytes.offsets.len() as i32, + created_at: timestamp(), + }; + Ok((chunked_bytes, record)) +} + +#[instrument(level = "debug", skip_all, err)] +async fn start_upload( + cloud_service: &Arc, + user_service: &Arc, + temp_storage: &Arc, + chunked_bytes: &ChunkedBytes, + upload_file: &UploadFileTable, + notifier: tokio::sync::broadcast::Sender, +) -> FlowyResult<()> { + let mut upload_file = upload_file.clone(); + if upload_file.upload_id.is_empty() { + // 1. create upload + trace!( + "[File] create upload for workspace: {}, parent_dir: {}, file_id: {}", + upload_file.workspace_id, + upload_file.parent_dir, + upload_file.file_id + ); + + let create_upload_resp = cloud_service + .create_upload( + &upload_file.workspace_id, + &upload_file.parent_dir, + &upload_file.file_id, + &upload_file.content_type, + ) + .await?; + // 2. update upload_id + let conn = user_service.sqlite_connection(user_service.user_id()?)?; + update_upload_file_upload_id( + conn, + &upload_file.workspace_id, + &upload_file.parent_dir, + &upload_file.file_id, + &create_upload_resp.upload_id, + )?; + + trace!( + "[File] {} update upload_id: {}", + upload_file.file_id, + create_upload_resp.upload_id + ); + // temporary store the upload_id + upload_file.upload_id = create_upload_resp.upload_id; + } + + let _ = notifier.send(UploadResult { + file_id: upload_file.file_id.clone(), + status: UploadStatus::InProgress, + }); + + // 3. start uploading parts + trace!( + "[File] {} start uploading parts: {}", + upload_file.file_id, + chunked_bytes.iter().count() + ); + let mut iter = chunked_bytes.iter().enumerate(); + let mut completed_parts = Vec::new(); + + while let Some((index, chunk_bytes)) = iter.next() { + let part_number = index as i32 + 1; + trace!( + "[File] {} uploading part: {}, len:{}KB", + upload_file.file_id, + part_number, + chunk_bytes.len() / 1000, + ); + // start uploading parts + match upload_part( + &cloud_service, + &user_service, + &upload_file.workspace_id, + &upload_file.parent_dir, + &upload_file.upload_id, + &upload_file.file_id, + part_number, + chunk_bytes.to_vec(), + ) + .await + { + Ok(resp) => { + trace!( + "[File] {} upload {} part success, total:{},", + upload_file.file_id, + part_number, + chunked_bytes.offsets.len() + ); + // gather completed part + completed_parts.push(CompletedPartRequest { + e_tag: resp.e_tag, + part_number: resp.part_num, + }); + }, + Err(err) => { + error!("[File] {} upload part failed: {}", upload_file.file_id, err); + return Err(err); + }, + } + } + + // mark it as completed + complete_upload( + &cloud_service, + &user_service, + temp_storage, + &upload_file, + completed_parts, + notifier, + ) + .await?; + + trace!("[File] {} upload completed", upload_file.file_id); + Ok(()) +} + +#[instrument(level = "debug", skip_all, err)] +async fn resume_upload( + cloud_service: &Arc, + user_service: &Arc, + temp_storage: &Arc, + upload_file: UploadFileTable, + parts: Vec, + notifier: tokio::sync::broadcast::Sender, +) -> FlowyResult<()> { + trace!( + "[File] resume upload for workspace: {}, parent_dir: {}, file_id: {}, local_file_path:{}", + upload_file.workspace_id, + upload_file.parent_dir, + upload_file.file_id, + upload_file.local_file_path + ); + + match ChunkedBytes::from_file(&upload_file.local_file_path, MIN_CHUNK_SIZE as i32).await { + Ok(mut chunked_bytes) => { + // When there were any parts already uploaded, skip those parts by setting the current offset. + chunked_bytes.set_current_offset(parts.len() as i32); + start_upload( + cloud_service, + user_service, + temp_storage, + &chunked_bytes, + &upload_file, + notifier, + ) + .await?; + }, + Err(err) => { + // + match err.kind() { + ErrorKind::NotFound => { + error!("[File] file not found: {}", upload_file.local_file_path); + if let Ok(uid) = user_service.user_id() { + if let Ok(conn) = user_service.sqlite_connection(uid) { + delete_upload_file(conn, &upload_file.upload_id)?; + } + } + }, + _ => { + error!("[File] read file failed: {}", err); + }, + } + }, + } + Ok(()) +} + +#[instrument(level = "debug", skip_all)] +async fn upload_part( + cloud_service: &Arc, + user_service: &Arc, + workspace_id: &str, + parent_dir: &str, + upload_id: &str, + file_id: &str, + part_number: i32, + body: Vec, +) -> Result { + let resp = cloud_service + .upload_part( + &workspace_id, + &parent_dir, + &upload_id, + &file_id, + part_number, + body, + ) + .await?; + + // save uploaded part to sqlite + let conn = user_service.sqlite_connection(user_service.user_id()?)?; + insert_upload_part( + conn, + &UploadFilePartTable { + upload_id: upload_id.to_string(), + e_tag: resp.e_tag.clone(), + part_num: resp.part_num, + }, + )?; + + Ok(resp) +} + +async fn complete_upload( + cloud_service: &Arc, + user_service: &Arc, + temp_storage: &Arc, + upload_file: &UploadFileTable, + parts: Vec, + notifier: tokio::sync::broadcast::Sender, +) -> Result<(), FlowyError> { + match cloud_service + .complete_upload( + &upload_file.workspace_id, + &upload_file.parent_dir, + &upload_file.upload_id, + &upload_file.file_id, + parts, + ) + .await + { + Ok(_) => { + info!("[File] completed upload file: {}", upload_file.upload_id); + trace!("[File] delete upload record from sqlite"); + let _ = notifier.send(UploadResult { + file_id: upload_file.file_id.clone(), + status: UploadStatus::Finish, + }); + + let conn = user_service.sqlite_connection(user_service.user_id()?)?; + delete_upload_file(conn, &upload_file.upload_id)?; + if let Err(err) = temp_storage + .delete_temp_file(&upload_file.local_file_path) + .await + { + error!("[File] delete temp file failed: {}", err); + } + }, + Err(err) => { + error!("[File] complete upload failed: {}", err); + }, + } + Ok(()) +} + +pub async fn object_from_disk( + workspace_id: &str, + local_file_path: &str, +) -> Result<(ObjectIdentity, ObjectValue), FlowyError> { + let ext = Path::new(local_file_path) + .extension() + .and_then(std::ffi::OsStr::to_str) + .unwrap_or("") + .to_owned(); + let mut file = tokio::fs::File::open(local_file_path).await?; + let mut content = Vec::new(); + let n = file.read_to_end(&mut content).await?; + info!("read {} bytes from file: {}", n, local_file_path); + let mime = mime_guess::from_path(local_file_path).first_or_octet_stream(); + let hash = fxhash::hash(&content); + + Ok(( + ObjectIdentity { + workspace_id: workspace_id.to_owned(), + file_id: hash.to_string(), + ext, + }, + ObjectValue { + raw: content.into(), + mime, + }, + )) +} diff --git a/frontend/rust-lib/flowy-storage/src/native/mod.rs b/frontend/rust-lib/flowy-storage/src/native/mod.rs deleted file mode 100644 index 777a8b08dc..0000000000 --- a/frontend/rust-lib/flowy-storage/src/native/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::{ObjectIdentity, ObjectValue}; -use flowy_error::FlowyError; -use std::path::Path; -use tokio::io::AsyncReadExt; -use tracing::info; - -pub async fn object_from_disk( - workspace_id: &str, - local_file_path: &str, -) -> Result<(ObjectIdentity, ObjectValue), FlowyError> { - let ext = Path::new(local_file_path) - .extension() - .and_then(std::ffi::OsStr::to_str) - .unwrap_or("") - .to_owned(); - let mut file = tokio::fs::File::open(local_file_path).await?; - let mut content = Vec::new(); - let n = file.read_to_end(&mut content).await?; - info!("read {} bytes from file: {}", n, local_file_path); - let mime = mime_guess::from_path(local_file_path).first_or_octet_stream(); - let hash = fxhash::hash(&content); - - Ok(( - ObjectIdentity { - workspace_id: workspace_id.to_owned(), - file_id: hash.to_string(), - ext, - }, - ObjectValue { - raw: content.into(), - mime, - }, - )) -} diff --git a/frontend/rust-lib/flowy-storage/src/sqlite_sql.rs b/frontend/rust-lib/flowy-storage/src/sqlite_sql.rs new file mode 100644 index 0000000000..339b3e4993 --- /dev/null +++ b/frontend/rust-lib/flowy-storage/src/sqlite_sql.rs @@ -0,0 +1,161 @@ +use flowy_error::{FlowyError, FlowyResult}; +use flowy_sqlite::schema::{upload_file_part, upload_file_table}; +use flowy_sqlite::{ + diesel, AsChangeset, BoolExpressionMethods, DBConnection, ExpressionMethods, Identifiable, + Insertable, OptionalExtension, QueryDsl, Queryable, RunQueryDsl, SqliteConnection, +}; +use tracing::warn; + +#[derive(Queryable, Insertable, AsChangeset, Identifiable, Debug, Clone)] +#[diesel(table_name = upload_file_table)] +#[diesel(primary_key(workspace_id, parent_dir, file_id))] +pub struct UploadFileTable { + pub workspace_id: String, + pub file_id: String, + pub parent_dir: String, + pub local_file_path: String, + pub content_type: String, + pub chunk_size: i32, + pub num_chunk: i32, + pub upload_id: String, + pub created_at: i64, +} + +#[derive(Queryable, Insertable, AsChangeset, Identifiable, Debug)] +#[diesel(table_name = upload_file_part)] +#[diesel(primary_key(upload_id, part_num))] +pub struct UploadFilePartTable { + pub upload_id: String, + pub e_tag: String, + pub part_num: i32, +} + +pub fn is_upload_file_exist( + conn: &mut SqliteConnection, + workspace_id: &str, + parent_dir: &str, + file_id: &str, +) -> FlowyResult { + let result = upload_file_table::dsl::upload_file_table + .filter( + upload_file_table::workspace_id + .eq(workspace_id) + .and(upload_file_table::parent_dir.eq(parent_dir)) + .and(upload_file_table::file_id.eq(file_id)), + ) + .first::(conn) + .optional()?; + Ok(result.is_some()) +} + +pub fn insert_upload_file( + mut conn: DBConnection, + upload_file: &UploadFileTable, +) -> FlowyResult<()> { + diesel::insert_into(upload_file_table::table) + .values(upload_file) + .execute(&mut *conn)?; + Ok(()) +} + +pub fn update_upload_file_upload_id( + mut conn: DBConnection, + workspace_id: &str, + parent_dir: &str, + file_id: &str, + upload_id: &str, +) -> FlowyResult<()> { + diesel::update( + upload_file_table::dsl::upload_file_table.filter( + upload_file_table::workspace_id + .eq(workspace_id) + .and(upload_file_table::parent_dir.eq(parent_dir)) + .and(upload_file_table::file_id.eq(file_id)), + ), + ) + .set(upload_file_table::upload_id.eq(upload_id)) + .execute(&mut *conn)?; + Ok(()) +} + +pub fn insert_upload_part( + mut conn: DBConnection, + upload_part: &UploadFilePartTable, +) -> FlowyResult<()> { + diesel::insert_into(upload_file_part::table) + .values(upload_part) + .execute(&mut *conn)?; + Ok(()) +} + +pub fn select_latest_upload_part( + mut conn: DBConnection, + upload_id: &str, +) -> FlowyResult> { + let result = upload_file_part::dsl::upload_file_part + .filter(upload_file_part::upload_id.eq(upload_id)) + .order(upload_file_part::part_num.desc()) + .first::(&mut *conn) + .optional()?; + Ok(result) +} + +pub fn select_upload_parts( + conn: &mut SqliteConnection, + upload_id: &str, +) -> FlowyResult> { + let results = upload_file_part::dsl::upload_file_part + .filter(upload_file_part::upload_id.eq(upload_id)) + .load::(conn)?; + Ok(results) +} + +pub fn batch_select_upload_file( + mut conn: DBConnection, + limit: i32, +) -> FlowyResult> { + let results = upload_file_table::dsl::upload_file_table + .order(upload_file_table::created_at.desc()) + .limit(limit.into()) + .load::(&mut conn)?; + Ok(results) +} + +pub fn select_upload_file( + conn: &mut SqliteConnection, + workspace_id: &str, + parent_dir: &str, + file_id: &str, +) -> FlowyResult> { + let result = upload_file_table::dsl::upload_file_table + .filter( + upload_file_table::workspace_id + .eq(workspace_id) + .and(upload_file_table::parent_dir.eq(parent_dir)) + .and(upload_file_table::file_id.eq(file_id)), + ) + .first::(conn) + .optional()?; + Ok(result) +} + +pub fn delete_upload_file(mut conn: DBConnection, upload_id: &str) -> FlowyResult<()> { + conn.immediate_transaction(|conn| { + diesel::delete( + upload_file_table::dsl::upload_file_table.filter(upload_file_table::upload_id.eq(upload_id)), + ) + .execute(&mut *conn)?; + + if let Err(err) = diesel::delete( + upload_file_part::dsl::upload_file_part.filter(upload_file_part::upload_id.eq(upload_id)), + ) + .execute(&mut *conn) + { + warn!("Failed to delete upload parts: {:?}", err) + } + + Ok::<_, FlowyError>(()) + })?; + + Ok(()) +} diff --git a/frontend/rust-lib/flowy-storage/src/uploader.rs b/frontend/rust-lib/flowy-storage/src/uploader.rs new file mode 100644 index 0000000000..5469c1d05b --- /dev/null +++ b/frontend/rust-lib/flowy-storage/src/uploader.rs @@ -0,0 +1,294 @@ +use crate::sqlite_sql::UploadFileTable; +use crate::uploader::UploadTask::BackgroundTask; +use flowy_storage_pub::chunked_byte::ChunkedBytes; +use flowy_storage_pub::storage::StorageService; +use lib_infra::box_any::BoxAny; +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::fmt::Display; +use std::sync::atomic::{AtomicBool, AtomicU8}; +use std::sync::{Arc, Weak}; +use std::time::Duration; +use tokio::sync::{watch, RwLock}; +use tracing::{info, trace}; + +#[derive(Clone)] +pub enum Signal { + Stop, + Proceed, + ProceedAfterSecs(u64), +} + +pub struct UploadTaskQueue { + tasks: RwLock>, + notifier: watch::Sender, +} + +impl UploadTaskQueue { + pub fn new(notifier: watch::Sender) -> Self { + Self { + tasks: Default::default(), + notifier, + } + } + pub async fn queue_task(&self, task: UploadTask) { + trace!("[File] Queued task: {}", task); + self.tasks.write().await.push(task); + let _ = self.notifier.send(Signal::Proceed); + } +} + +pub struct FileUploader { + storage_service: Arc, + queue: Arc, + max_uploads: u8, + current_uploads: AtomicU8, + pause_sync: AtomicBool, +} + +impl Drop for FileUploader { + fn drop(&mut self) { + let _ = self.queue.notifier.send(Signal::Stop); + } +} + +impl FileUploader { + pub fn new(storage_service: Arc, queue: Arc) -> Self { + Self { + storage_service, + queue, + max_uploads: 3, + current_uploads: Default::default(), + pause_sync: Default::default(), + } + } + + pub async fn queue_tasks(&self, tasks: Vec) { + let mut queue_lock = self.queue.tasks.write().await; + for task in tasks { + queue_lock.push(task); + } + let _ = self.queue.notifier.send(Signal::Proceed); + } + + pub fn pause(&self) { + self + .pause_sync + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + pub fn resume(&self) { + self + .pause_sync + .store(false, std::sync::atomic::Ordering::SeqCst); + let _ = self.queue.notifier.send(Signal::ProceedAfterSecs(3)); + } + + pub async fn process_next(&self) -> Option<()> { + // Do not proceed if the uploader is paused. + if self.pause_sync.load(std::sync::atomic::Ordering::Relaxed) { + return None; + } + + trace!( + "[File] Max concurrent uploads: {}, current: {}", + self.max_uploads, + self + .current_uploads + .load(std::sync::atomic::Ordering::SeqCst) + ); + + if self + .current_uploads + .load(std::sync::atomic::Ordering::SeqCst) + >= self.max_uploads + { + // If the current uploads count is greater than or equal to the max uploads, do not proceed. + let _ = self.queue.notifier.send(Signal::ProceedAfterSecs(10)); + return None; + } + + let task = self.queue.tasks.write().await.pop()?; + if task.retry_count() > 5 { + // If the task has been retried more than 5 times, we should not retry it anymore. + let _ = self.queue.notifier.send(Signal::ProceedAfterSecs(2)); + return None; + } + + // increment the current uploads count + self + .current_uploads + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + match task { + UploadTask::Task { + chunks, + record, + mut retry_count, + } => { + let record = BoxAny::new(record); + if let Err(err) = self.storage_service.start_upload(&chunks, &record).await { + info!( + "Failed to upload file: {}, retry_count:{}", + err, retry_count + ); + + let record = record.unbox_or_error().unwrap(); + retry_count += 1; + self.queue.tasks.write().await.push(UploadTask::Task { + chunks, + record, + retry_count, + }); + } + }, + UploadTask::BackgroundTask { + workspace_id, + parent_dir, + file_id, + created_at, + mut retry_count, + } => { + if let Err(err) = self + .storage_service + .resume_upload(&workspace_id, &parent_dir, &file_id) + .await + { + info!( + "Failed to resume upload file: {}, retry_count:{}", + err, retry_count + ); + retry_count += 1; + self.queue.tasks.write().await.push(BackgroundTask { + workspace_id, + parent_dir, + file_id, + created_at, + retry_count, + }); + } + }, + } + self + .current_uploads + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + let _ = self.queue.notifier.send(Signal::ProceedAfterSecs(2)); + None + } +} + +pub struct FileUploaderRunner; + +impl FileUploaderRunner { + pub async fn run(weak_uploader: Weak, mut notifier: watch::Receiver) { + loop { + // stops the runner if the notifier was closed. + if notifier.changed().await.is_err() { + break; + } + + if let Some(uploader) = weak_uploader.upgrade() { + let value = notifier.borrow().clone(); + match value { + Signal::Stop => break, + Signal::Proceed => { + tokio::spawn(async move { + uploader.process_next().await; + }); + }, + Signal::ProceedAfterSecs(secs) => { + tokio::time::sleep(Duration::from_secs(secs)).await; + tokio::spawn(async move { + uploader.process_next().await; + }); + }, + } + } else { + break; + } + } + } +} + +pub enum UploadTask { + Task { + chunks: ChunkedBytes, + record: UploadFileTable, + retry_count: u8, + }, + BackgroundTask { + workspace_id: String, + file_id: String, + parent_dir: String, + created_at: i64, + retry_count: u8, + }, +} + +impl UploadTask { + pub fn retry_count(&self) -> u8 { + match self { + Self::Task { retry_count, .. } => *retry_count, + Self::BackgroundTask { retry_count, .. } => *retry_count, + } + } +} + +impl Display for UploadTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Task { record, .. } => write!(f, "Task: {}", record.file_id), + Self::BackgroundTask { file_id, .. } => write!(f, "BackgroundTask: {}", file_id), + } + } +} +impl Eq for UploadTask {} + +impl PartialEq for UploadTask { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Task { record: lhs, .. }, Self::Task { record: rhs, .. }) => { + lhs.local_file_path == rhs.local_file_path + }, + ( + Self::BackgroundTask { + workspace_id: l_workspace_id, + file_id: l_file_id, + .. + }, + Self::BackgroundTask { + workspace_id: r_workspace_id, + file_id: r_file_id, + .. + }, + ) => l_workspace_id == r_workspace_id && l_file_id == r_file_id, + _ => false, + } + } +} + +impl PartialOrd for UploadTask { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for UploadTask { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (Self::Task { record: lhs, .. }, Self::Task { record: rhs, .. }) => { + lhs.created_at.cmp(&rhs.created_at) + }, + (_, Self::Task { .. }) => Ordering::Less, + (Self::Task { .. }, _) => Ordering::Greater, + ( + Self::BackgroundTask { + created_at: lhs, .. + }, + Self::BackgroundTask { + created_at: rhs, .. + }, + ) => lhs.cmp(rhs), + } + } +} diff --git a/frontend/rust-lib/flowy-storage/src/wasm/mod.rs b/frontend/rust-lib/flowy-storage/src/wasm/mod.rs deleted file mode 100644 index 8d4d3b1bfc..0000000000 --- a/frontend/rust-lib/flowy-storage/src/wasm/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::{ObjectIdentity, ObjectValue}; -use flowy_error::FlowyError; - -pub async fn object_from_disk( - _workspace_id: &str, - _local_file_path: &str, -) -> Result<(ObjectIdentity, ObjectValue), FlowyError> { - Err( - FlowyError::not_support() - .with_context(format!("object_from_disk is not implemented for wasm32")), - ) -} diff --git a/frontend/rust-lib/flowy-storage/tests/multiple_part_upload_test.rs b/frontend/rust-lib/flowy-storage/tests/multiple_part_upload_test.rs new file mode 100644 index 0000000000..42facd25a4 --- /dev/null +++ b/frontend/rust-lib/flowy-storage/tests/multiple_part_upload_test.rs @@ -0,0 +1,181 @@ +use flowy_sqlite::Database; +use flowy_storage::sqlite_sql::{ + batch_select_upload_file, delete_upload_file, insert_upload_file, insert_upload_part, + select_latest_upload_part, select_upload_parts, UploadFilePartTable, UploadFileTable, +}; +use flowy_storage_pub::chunked_byte::{ChunkedBytes, MIN_CHUNK_SIZE}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; +use std::env::temp_dir; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +pub fn test_database() -> (Database, PathBuf) { + let db_path = temp_dir().join(&format!("test-{}.db", generate_random_string(8))); + (flowy_sqlite::init(&db_path).unwrap(), db_path) +} + +#[tokio::test] +async fn test_insert_new_upload() { + let (db, _) = test_database(); + + let workspace_id = uuid::Uuid::new_v4().to_string(); + + // test insert one upload file record + let mut upload_ids = vec![]; + for _i in 0..5 { + let upload_id = uuid::Uuid::new_v4().to_string(); + let local_file_path = create_temp_file_with_random_content(8 * 1024 * 1024).unwrap(); + let upload_file = + create_upload_file_record(workspace_id.clone(), upload_id.clone(), local_file_path).await; + upload_ids.push(upload_file.upload_id.clone()); + + // insert + let conn = db.get_connection().unwrap(); + insert_upload_file(conn, &upload_file).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + } + upload_ids.reverse(); + + // select + let conn = db.get_connection().unwrap(); + let records = batch_select_upload_file(conn, 100).unwrap(); + + assert_eq!(records.len(), 5); + // compare the upload id order is the same as upload_ids + for i in 0..5 { + assert_eq!(records[i].upload_id, upload_ids[i]); + + // delete + let conn = db.get_connection().unwrap(); + delete_upload_file(conn, &records[i].upload_id).unwrap(); + } + + let conn = db.get_connection().unwrap(); + let records = batch_select_upload_file(conn, 100).unwrap(); + assert!(records.is_empty()); +} + +#[tokio::test] +async fn test_upload_part_test() { + let (db, _) = test_database(); + + let workspace_id = uuid::Uuid::new_v4().to_string(); + + // test insert one upload file record + let upload_id = uuid::Uuid::new_v4().to_string(); + let local_file_path = create_temp_file_with_random_content(20 * 1024 * 1024).unwrap(); + let upload_file = + create_upload_file_record(workspace_id.clone(), upload_id.clone(), local_file_path).await; + + // insert + let conn = db.get_connection().unwrap(); + insert_upload_file(conn, &upload_file).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + + // insert uploaded part 1 + let part = UploadFilePartTable { + upload_id: upload_id.clone(), + e_tag: "1".to_string(), + part_num: 1, + }; + let conn = db.get_connection().unwrap(); + insert_upload_part(conn, &part).unwrap(); + + // insert uploaded part 2 + let part = UploadFilePartTable { + upload_id: upload_id.clone(), + e_tag: "2".to_string(), + part_num: 2, + }; + let conn = db.get_connection().unwrap(); + insert_upload_part(conn, &part).unwrap(); + + // get latest part + let conn = db.get_connection().unwrap(); + let part = select_latest_upload_part(conn, &upload_id) + .unwrap() + .unwrap(); + assert_eq!(part.part_num, 2); + + // get all existing parts + let mut conn = db.get_connection().unwrap(); + let parts = select_upload_parts(&mut *conn, &upload_id).unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0].part_num, 1); + assert_eq!(parts[1].part_num, 2); + + // delete upload file and then all existing parts will be deleted + let conn = db.get_connection().unwrap(); + delete_upload_file(conn, &upload_id).unwrap(); + + let mut conn = db.get_connection().unwrap(); + let parts = select_upload_parts(&mut *conn, &upload_id).unwrap(); + assert!(parts.is_empty()) +} + +pub fn generate_random_string(len: usize) -> String { + let rng = thread_rng(); + rng + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} + +fn create_temp_file_with_random_content( + size_in_bytes: usize, +) -> Result> { + // Generate a random string of the specified size + let content: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(size_in_bytes) + .map(char::from) + .collect(); + + // Create a temporary file path + let file_path = std::env::temp_dir().join("test.txt"); + + // Write the content to the temporary file + let mut file = File::create(&file_path)?; + file.write_all(content.as_bytes())?; + + // Return the file path + Ok(file_path.to_str().unwrap().to_string()) +} + +pub async fn create_upload_file_record( + workspace_id: String, + upload_id: String, + local_file_path: String, +) -> UploadFileTable { + // Create ChunkedBytes from file + let chunked_bytes = ChunkedBytes::from_file(&local_file_path, MIN_CHUNK_SIZE as i32) + .await + .unwrap(); + + // Determine content type + let content_type = mime_guess::from_path(&local_file_path) + .first_or_octet_stream() + .to_string(); + + // Calculate file ID + let file_id = fxhash::hash(&chunked_bytes.data).to_string(); + + // Create UploadFileTable record + let upload_file = UploadFileTable { + workspace_id, + file_id, + upload_id, + parent_dir: "test".to_string(), + local_file_path, + content_type, + chunk_size: MIN_CHUNK_SIZE as i32, + num_chunk: chunked_bytes.offsets.len() as i32, + created_at: chrono::Utc::now().timestamp(), + }; + + upload_file +} diff --git a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs index c9d779a232..dc7b9c1b9c 100644 --- a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs +++ b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs @@ -88,6 +88,10 @@ impl AuthenticateUser { PathBuf::from(self.user_paths.user_data_dir(uid)).join("indexes") } + pub fn get_application_root_dir(&self) -> &str { + self.user_paths.root() + } + pub fn close_db(&self) -> FlowyResult<()> { let session = self.get_session()?; info!("Close db for user: {}", session.user_id); diff --git a/frontend/rust-lib/flowy-user/src/services/entities.rs b/frontend/rust-lib/flowy-user/src/services/entities.rs index 831ef10751..4e034b3bdb 100644 --- a/frontend/rust-lib/flowy-user/src/services/entities.rs +++ b/frontend/rust-lib/flowy-user/src/services/entities.rs @@ -63,6 +63,11 @@ impl UserPaths { pub(crate) fn user_data_dir(&self, uid: i64) -> String { format!("{}/{}", self.root, uid) } + + /// The root directory of the application + pub(crate) fn root(&self) -> &str { + &self.root + } } impl UserDBPath for UserPaths {