From 058eeec932a53e29a4826061fe2610156a88acd5 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:54:45 +0800 Subject: [PATCH] feat: implement file storage using appflowy cloud (#3675) * feat: implement file storage using appflowy cloud * chore: clippy --- frontend/appflowy_tauri/src-tauri/Cargo.lock | 34 ++++---------- frontend/appflowy_tauri/src-tauri/Cargo.toml | 2 +- frontend/rust-lib/Cargo.lock | 14 +++--- frontend/rust-lib/Cargo.toml | 2 +- .../tests/document/supabase_test/file_test.rs | 17 +++++-- .../flowy-error/src/impl_from/cloud.rs | 1 - .../src/af_cloud/impls/file_storage.rs | 47 +++++++++++++------ .../tests/supabase_test/file_test.rs | 6 +-- frontend/rust-lib/flowy-storage/src/lib.rs | 12 ++++- 9 files changed, 79 insertions(+), 56 deletions(-) diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.lock b/frontend/appflowy_tauri/src-tauri/Cargo.lock index 0cc046cf77..1bb8716dbc 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.lock +++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock @@ -454,7 +454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive", - "hashbrown 0.12.3", + "hashbrown 0.13.2", ] [[package]] @@ -762,7 +762,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "bytes", @@ -1291,7 +1291,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa 1.0.6", - "phf 0.11.2", + "phf 0.8.0", "smallvec", ] @@ -1437,7 +1437,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "chrono", @@ -1976,7 +1976,7 @@ dependencies = [ "collab-integrate", "csv", "dashmap", - "fancy-regex 0.10.0", + "fancy-regex 0.11.0", "flowy-codegen", "flowy-database-deps", "flowy-derive", @@ -2778,7 +2778,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "futures-util", @@ -2794,7 +2794,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "reqwest", @@ -3227,7 +3227,7 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "reqwest", @@ -4268,7 +4268,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ - "phf_macros 0.11.2", "phf_shared 0.11.2", ] @@ -4360,19 +4359,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "phf_macros" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" -dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", - "proc-macro2", - "quote", - "syn 2.0.29", -] - [[package]] name = "phf_shared" version = "0.8.0" @@ -4876,7 +4862,7 @@ dependencies = [ [[package]] name = "realtime-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "bytes", "collab", @@ -5598,7 +5584,7 @@ dependencies = [ [[package]] name = "shared_entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "database-entity", diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.toml b/frontend/appflowy_tauri/src-tauri/Cargo.toml index 10b069c01c..47d9874447 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.toml +++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml @@ -38,7 +38,7 @@ custom-protocol = ["tauri/custom-protocol"] # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6faefb0" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "5a231fda" } # Please use the following script to update collab. # Working directory: frontend # diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index c24d7e49b8..ba7077075e 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -660,7 +660,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "bytes", @@ -1264,7 +1264,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "chrono", @@ -2438,7 +2438,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "futures-util", @@ -2454,7 +2454,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "reqwest", @@ -2812,7 +2812,7 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "reqwest", @@ -4203,7 +4203,7 @@ dependencies = [ [[package]] name = "realtime-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "bytes", "collab", @@ -4824,7 +4824,7 @@ dependencies = [ [[package]] name = "shared_entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7" dependencies = [ "anyhow", "database-entity", diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml index 6914e30454..4496ae338d 100644 --- a/frontend/rust-lib/Cargo.toml +++ b/frontend/rust-lib/Cargo.toml @@ -82,7 +82,7 @@ incremental = false # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6faefb0" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "5a231fda" } # Please use the following script to update collab. # Working directory: frontend # diff --git a/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs b/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs index 7430760b96..02c0e870c7 100644 --- a/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs +++ b/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs @@ -12,6 +12,7 @@ use crate::document::supabase_test::helper::FlowySupabaseDocumentTest; #[tokio::test] async fn supabase_document_upload_text_file_test() { if let Some(test) = FlowySupabaseDocumentTest::new().await { + let workspace_id = test.get_current_workspace().await.workspace.id; let storage_service = test .document_manager .get_file_storage_service() @@ -19,6 +20,7 @@ async fn supabase_document_upload_text_file_test() { .unwrap(); let object = StorageObject::from_bytes( + &workspace_id, &Uuid::new_v4().to_string(), "hello world".as_bytes(), "text/plain".to_string(), @@ -41,6 +43,7 @@ async fn supabase_document_upload_text_file_test() { #[tokio::test] async fn supabase_document_upload_zip_file_test() { if let Some(test) = FlowySupabaseDocumentTest::new().await { + let workspace_id = test.get_current_workspace().await.workspace.id; let storage_service = test .document_manager .get_file_storage_service() @@ -48,8 +51,11 @@ async fn supabase_document_upload_zip_file_test() { .unwrap(); // Upload zip file - let object = - StorageObject::from_file(&Uuid::new_v4().to_string(), "./tests/asset/test.txt.zip"); + let object = StorageObject::from_file( + &workspace_id, + &Uuid::new_v4().to_string(), + "./tests/asset/test.txt.zip", + ); let url = storage_service.create_object(object).await.unwrap(); // Read zip file @@ -79,6 +85,7 @@ async fn supabase_document_upload_zip_file_test() { #[tokio::test] async fn supabase_document_upload_image_test() { if let Some(test) = FlowySupabaseDocumentTest::new().await { + let workspace_id = test.get_current_workspace().await.workspace.id; let storage_service = test .document_manager .get_file_storage_service() @@ -86,7 +93,11 @@ async fn supabase_document_upload_image_test() { .unwrap(); // Upload zip file - let object = StorageObject::from_file(&Uuid::new_v4().to_string(), "./tests/asset/logo.png"); + let object = StorageObject::from_file( + &workspace_id, + &Uuid::new_v4().to_string(), + "./tests/asset/logo.png", + ); let url = storage_service.create_object(object).await.unwrap(); let image_data = storage_service diff --git a/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs b/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs index 04b6ef2f59..97221c886f 100644 --- a/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs +++ b/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs @@ -8,7 +8,6 @@ impl From for FlowyError { client_api::error::ErrorCode::Ok => ErrorCode::Internal, client_api::error::ErrorCode::Unhandled => ErrorCode::Internal, client_api::error::ErrorCode::RecordNotFound => ErrorCode::RecordNotFound, - client_api::error::ErrorCode::FileNotFound => ErrorCode::RecordNotFound, client_api::error::ErrorCode::RecordAlreadyExists => ErrorCode::RecordAlreadyExists, client_api::error::ErrorCode::InvalidEmail => ErrorCode::EmailFormatInvalid, client_api::error::ErrorCode::InvalidPassword => ErrorCode::PasswordFormatInvalid, diff --git a/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs b/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs index 8710377eac..507a26eabb 100644 --- a/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs +++ b/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs @@ -1,18 +1,18 @@ use bytes::Bytes; +use tokio::fs::File; +use tokio::io::AsyncReadExt; + use flowy_error::FlowyError; -use flowy_storage::{FileStorageService, StorageObject}; +use flowy_storage::{FileStorageService, ObjectValue, StorageObject}; use lib_infra::future::FutureResult; use crate::af_cloud::AFServer; -pub struct AFCloudFileStorageServiceImpl { - #[allow(dead_code)] - client: T, -} +pub struct AFCloudFileStorageServiceImpl(pub T); impl AFCloudFileStorageServiceImpl { pub fn new(client: T) -> Self { - Self { client } + Self(client) } } @@ -20,24 +20,43 @@ impl FileStorageService for AFCloudFileStorageServiceImpl where T: AFServer, { - fn create_object(&self, _object: StorageObject) -> FutureResult { + fn create_object(&self, object: StorageObject) -> FutureResult { + let try_get_client = self.0.try_get_client(); FutureResult::new(async move { - // TODO - Ok("".to_owned()) + let client = try_get_client?; + + match object.value { + ObjectValue::File { file_path } => { + let mut file = File::open(&file_path).await?; + let mime = mime_guess::from_path(file_path) + .first_or_octet_stream() + .to_string(); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).await?; + Ok(client.put_file(&object.workspace_id, buffer, mime).await?) + }, + ObjectValue::Bytes { bytes, mime } => { + Ok(client.put_file(&object.workspace_id, bytes, mime).await?) + }, + } }) } - fn delete_object_by_url(&self, _object_url: String) -> FutureResult<(), FlowyError> { + fn delete_object_by_url(&self, object_url: String) -> FutureResult<(), FlowyError> { + let try_get_client = self.0.try_get_client(); FutureResult::new(async move { - // TODO + let client = try_get_client?; + client.delete_file(&object_url).await?; Ok(()) }) } - fn get_object_by_url(&self, _object_url: String) -> FutureResult { + fn get_object_by_url(&self, object_url: String) -> FutureResult { + let try_get_client = self.0.try_get_client(); FutureResult::new(async move { - // TODO - Ok(Bytes::new()) + let client = try_get_client?; + let bytes = client.get_file(&object_url).await?; + Ok(bytes) }) } } diff --git a/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs b/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs index de643ac661..d388182776 100644 --- a/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs +++ b/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs @@ -13,7 +13,7 @@ async fn supabase_get_object_test() { let service = file_storage_service(); let file_name = format!("test-{}.txt", Uuid::new_v4()); - let object = StorageObject::from_file(&file_name, "tests/test.txt"); + let object = StorageObject::from_file("1", &file_name, "tests/test.txt"); // Upload a file let url = service @@ -42,7 +42,7 @@ async fn supabase_upload_image_test() { let service = file_storage_service(); let file_name = format!("image-{}.png", Uuid::new_v4()); - let object = StorageObject::from_file(&file_name, "tests/logo.png"); + let object = StorageObject::from_file("1", &file_name, "tests/logo.png"); // Upload a file let url = service @@ -65,7 +65,7 @@ async fn supabase_delete_object_test() { let service = file_storage_service(); let file_name = format!("test-{}.txt", Uuid::new_v4()); - let object = StorageObject::from_file(&file_name, "tests/test.txt"); + let object = StorageObject::from_file("1", &file_name, "tests/test.txt"); let url = service.create_object(object).await.unwrap(); let result = service.get_object_by_url(url.clone()).await; diff --git a/frontend/rust-lib/flowy-storage/src/lib.rs b/frontend/rust-lib/flowy-storage/src/lib.rs index dd1667fc20..5071933b76 100644 --- a/frontend/rust-lib/flowy-storage/src/lib.rs +++ b/frontend/rust-lib/flowy-storage/src/lib.rs @@ -4,6 +4,7 @@ use flowy_error::FlowyError; use lib_infra::future::FutureResult; pub struct StorageObject { + pub workspace_id: String, pub file_name: String, pub value: ObjectValue, } @@ -16,8 +17,9 @@ impl StorageObject { /// * `name`: The name of the storage object. /// * `file_path`: The file path to the storage object's data. /// - pub fn from_file(file_name: &str, file_path: T) -> Self { + pub fn from_file(workspace_id: &str, file_name: &str, file_path: T) -> Self { Self { + workspace_id: workspace_id.to_string(), file_name: file_name.to_string(), value: ObjectValue::File { file_path: file_path.to_string(), @@ -33,9 +35,15 @@ impl StorageObject { /// * `bytes`: The byte data of the storage object. /// * `mime`: The MIME type of the storage object. /// - pub fn from_bytes>(file_name: &str, bytes: B, mime: String) -> Self { + pub fn from_bytes>( + workspace_id: &str, + file_name: &str, + bytes: B, + mime: String, + ) -> Self { let bytes = bytes.into(); Self { + workspace_id: workspace_id.to_string(), file_name: file_name.to_string(), value: ObjectValue::Bytes { bytes, mime }, }