feat: implement file storage using appflowy cloud (#3675)

* feat: implement file storage using appflowy cloud * chore: clippy
2024-08-30 18:12:39 +00:00 · 2023-10-12 09:54:45 +08:00 · 2023-10-12 09:54:45 +08:00 · 058eeec932
commit 058eeec932
parent 8e10cba8e5
9 changed files with 79 additions and 56 deletions
--- a/frontend/appflowy_tauri/src-tauri/Cargo.lock
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock
@ -454,7 +454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
 dependencies = [
 "borsh-derive",
- "hashbrown 0.12.3",
+ "hashbrown 0.13.2",
 ]

 [[package]]
@ -762,7 +762,7 @@ dependencies = [
 [[package]]
 name = "client-api"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "bytes",
@ -1291,7 +1291,7 @@ dependencies = [
 "cssparser-macros",
 "dtoa-short",
 "itoa 1.0.6",
- "phf 0.11.2",
+ "phf 0.8.0",
 "smallvec",
 ]

@ -1437,7 +1437,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
 [[package]]
 name = "database-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "chrono",
@ -1976,7 +1976,7 @@ dependencies = [
 "collab-integrate",
 "csv",
 "dashmap",
- "fancy-regex 0.10.0",
+ "fancy-regex 0.11.0",
 "flowy-codegen",
 "flowy-database-deps",
 "flowy-derive",
@ -2778,7 +2778,7 @@ dependencies = [
 [[package]]
 name = "gotrue"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "futures-util",
@ -2794,7 +2794,7 @@ dependencies = [
 [[package]]
 name = "gotrue-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "reqwest",
@ -3227,7 +3227,7 @@ dependencies = [
 [[package]]
 name = "infra"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "reqwest",
@ -4268,7 +4268,6 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
 dependencies = [
- "phf_macros 0.11.2",
 "phf_shared 0.11.2",
 ]

@ -4360,19 +4359,6 @@ dependencies = [
 "syn 1.0.109",
 ]

-[[package]]
-name = "phf_macros"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
-dependencies = [
- "phf_generator 0.11.2",
- "phf_shared 0.11.2",
- "proc-macro2",
- "quote",
- "syn 2.0.29",
-]
-
 [[package]]
 name = "phf_shared"
 version = "0.8.0"
@ -4876,7 +4862,7 @@ dependencies = [
 [[package]]
 name = "realtime-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "bytes",
 "collab",
@ -5598,7 +5584,7 @@ dependencies = [
 [[package]]
 name = "shared_entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "database-entity",
--- a/frontend/appflowy_tauri/src-tauri/Cargo.toml
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml
@ -38,7 +38,7 @@ custom-protocol = ["tauri/custom-protocol"]
 # Run the script:
 # scripts/tool/update_client_api_rev.sh  new_rev_id
 # ⚠️⚠️⚠️️
-client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6faefb0" }
+client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "5a231fda" }
 # Please use the following script to update collab.
 # Working directory: frontend
 #
--- a/frontend/rust-lib/Cargo.lock
+++ b/frontend/rust-lib/Cargo.lock
@ -660,7 +660,7 @@ dependencies = [
 [[package]]
 name = "client-api"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "bytes",
@ -1264,7 +1264,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
 [[package]]
 name = "database-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "chrono",
@ -2438,7 +2438,7 @@ dependencies = [
 [[package]]
 name = "gotrue"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "futures-util",
@ -2454,7 +2454,7 @@ dependencies = [
 [[package]]
 name = "gotrue-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "reqwest",
@ -2812,7 +2812,7 @@ dependencies = [
 [[package]]
 name = "infra"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "reqwest",
@ -4203,7 +4203,7 @@ dependencies = [
 [[package]]
 name = "realtime-entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "bytes",
 "collab",
@ -4824,7 +4824,7 @@ dependencies = [
 [[package]]
 name = "shared_entity"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6faefb0#6faefb083fe0a127045ec57431f126e499f4a687"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=5a231fda#5a231fdadba514d891e914e95c90bfcca7e042b7"
 dependencies = [
 "anyhow",
 "database-entity",
--- a/frontend/rust-lib/Cargo.toml
+++ b/frontend/rust-lib/Cargo.toml
@ -82,7 +82,7 @@ incremental = false
 # Run the script:
 # scripts/tool/update_client_api_rev.sh  new_rev_id
 # ⚠️⚠️⚠️️
-client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6faefb0" }
+client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "5a231fda" }
 # Please use the following script to update collab.
 # Working directory: frontend
 #
--- a/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs
+++ b/frontend/rust-lib/event-integration/tests/document/supabase_test/file_test.rs
@ -12,6 +12,7 @@ use crate::document::supabase_test::helper::FlowySupabaseDocumentTest;
 #[tokio::test]
 async fn supabase_document_upload_text_file_test() {
  if let Some(test) = FlowySupabaseDocumentTest::new().await {
+    let workspace_id = test.get_current_workspace().await.workspace.id;
    let storage_service = test
      .document_manager
      .get_file_storage_service()
@ -19,6 +20,7 @@ async fn supabase_document_upload_text_file_test() {
      .unwrap();

    let object = StorageObject::from_bytes(
+      &workspace_id,
      &Uuid::new_v4().to_string(),
      "hello world".as_bytes(),
      "text/plain".to_string(),
@ -41,6 +43,7 @@ async fn supabase_document_upload_text_file_test() {
 #[tokio::test]
 async fn supabase_document_upload_zip_file_test() {
  if let Some(test) = FlowySupabaseDocumentTest::new().await {
+    let workspace_id = test.get_current_workspace().await.workspace.id;
    let storage_service = test
      .document_manager
      .get_file_storage_service()
@ -48,8 +51,11 @@ async fn supabase_document_upload_zip_file_test() {
      .unwrap();

    // Upload zip file
-    let object =
-      StorageObject::from_file(&Uuid::new_v4().to_string(), "./tests/asset/test.txt.zip");
+    let object = StorageObject::from_file(
+      &workspace_id,
+      &Uuid::new_v4().to_string(),
+      "./tests/asset/test.txt.zip",
+    );
    let url = storage_service.create_object(object).await.unwrap();

    // Read zip file
@ -79,6 +85,7 @@ async fn supabase_document_upload_zip_file_test() {
 #[tokio::test]
 async fn supabase_document_upload_image_test() {
  if let Some(test) = FlowySupabaseDocumentTest::new().await {
+    let workspace_id = test.get_current_workspace().await.workspace.id;
    let storage_service = test
      .document_manager
      .get_file_storage_service()
@ -86,7 +93,11 @@ async fn supabase_document_upload_image_test() {
      .unwrap();

    // Upload zip file
-    let object = StorageObject::from_file(&Uuid::new_v4().to_string(), "./tests/asset/logo.png");
+    let object = StorageObject::from_file(
+      &workspace_id,
+      &Uuid::new_v4().to_string(),
+      "./tests/asset/logo.png",
+    );
    let url = storage_service.create_object(object).await.unwrap();

    let image_data = storage_service
--- a/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs
+++ b/frontend/rust-lib/flowy-error/src/impl_from/cloud.rs
@ -8,7 +8,6 @@ impl From<AppError> for FlowyError {
      client_api::error::ErrorCode::Ok => ErrorCode::Internal,
      client_api::error::ErrorCode::Unhandled => ErrorCode::Internal,
      client_api::error::ErrorCode::RecordNotFound => ErrorCode::RecordNotFound,
-      client_api::error::ErrorCode::FileNotFound => ErrorCode::RecordNotFound,
      client_api::error::ErrorCode::RecordAlreadyExists => ErrorCode::RecordAlreadyExists,
      client_api::error::ErrorCode::InvalidEmail => ErrorCode::EmailFormatInvalid,
      client_api::error::ErrorCode::InvalidPassword => ErrorCode::PasswordFormatInvalid,
--- a/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs
+++ b/frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs
@ -1,18 +1,18 @@
 use bytes::Bytes;
+use tokio::fs::File;
+use tokio::io::AsyncReadExt;
+
 use flowy_error::FlowyError;
-use flowy_storage::{FileStorageService, StorageObject};
+use flowy_storage::{FileStorageService, ObjectValue, StorageObject};
 use lib_infra::future::FutureResult;

 use crate::af_cloud::AFServer;

-pub struct AFCloudFileStorageServiceImpl<T> {
-  #[allow(dead_code)]
-  client: T,
-}
+pub struct AFCloudFileStorageServiceImpl<T>(pub T);

 impl<T> AFCloudFileStorageServiceImpl<T> {
  pub fn new(client: T) -> Self {
-    Self { client }
+    Self(client)
  }
 }

@ -20,24 +20,43 @@ impl<T> FileStorageService for AFCloudFileStorageServiceImpl<T>
 where
  T: AFServer,
 {
-  fn create_object(&self, _object: StorageObject) -> FutureResult<String, FlowyError> {
+  fn create_object(&self, object: StorageObject) -> FutureResult<String, FlowyError> {
+    let try_get_client = self.0.try_get_client();
    FutureResult::new(async move {
-      // TODO
-      Ok("".to_owned())
+      let client = try_get_client?;
+
+      match object.value {
+        ObjectValue::File { file_path } => {
+          let mut file = File::open(&file_path).await?;
+          let mime = mime_guess::from_path(file_path)
+            .first_or_octet_stream()
+            .to_string();
+          let mut buffer = Vec::new();
+          file.read_to_end(&mut buffer).await?;
+          Ok(client.put_file(&object.workspace_id, buffer, mime).await?)
+        },
+        ObjectValue::Bytes { bytes, mime } => {
+          Ok(client.put_file(&object.workspace_id, bytes, mime).await?)
+        },
+      }
    })
  }

-  fn delete_object_by_url(&self, _object_url: String) -> FutureResult<(), FlowyError> {
+  fn delete_object_by_url(&self, object_url: String) -> FutureResult<(), FlowyError> {
+    let try_get_client = self.0.try_get_client();
    FutureResult::new(async move {
-      // TODO
+      let client = try_get_client?;
+      client.delete_file(&object_url).await?;
      Ok(())
    })
  }

-  fn get_object_by_url(&self, _object_url: String) -> FutureResult<Bytes, FlowyError> {
+  fn get_object_by_url(&self, object_url: String) -> FutureResult<Bytes, FlowyError> {
+    let try_get_client = self.0.try_get_client();
    FutureResult::new(async move {
-      // TODO
-      Ok(Bytes::new())
+      let client = try_get_client?;
+      let bytes = client.get_file(&object_url).await?;
+      Ok(bytes)
    })
  }
 }
--- a/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs
+++ b/frontend/rust-lib/flowy-server/tests/supabase_test/file_test.rs
@ -13,7 +13,7 @@ async fn supabase_get_object_test() {

  let service = file_storage_service();
  let file_name = format!("test-{}.txt", Uuid::new_v4());
-  let object = StorageObject::from_file(&file_name, "tests/test.txt");
+  let object = StorageObject::from_file("1", &file_name, "tests/test.txt");

  // Upload a file
  let url = service
@ -42,7 +42,7 @@ async fn supabase_upload_image_test() {

  let service = file_storage_service();
  let file_name = format!("image-{}.png", Uuid::new_v4());
-  let object = StorageObject::from_file(&file_name, "tests/logo.png");
+  let object = StorageObject::from_file("1", &file_name, "tests/logo.png");

  // Upload a file
  let url = service
@ -65,7 +65,7 @@ async fn supabase_delete_object_test() {

  let service = file_storage_service();
  let file_name = format!("test-{}.txt", Uuid::new_v4());
-  let object = StorageObject::from_file(&file_name, "tests/test.txt");
+  let object = StorageObject::from_file("1", &file_name, "tests/test.txt");
  let url = service.create_object(object).await.unwrap();

  let result = service.get_object_by_url(url.clone()).await;
--- a/frontend/rust-lib/flowy-storage/src/lib.rs
+++ b/frontend/rust-lib/flowy-storage/src/lib.rs
@ -4,6 +4,7 @@ use flowy_error::FlowyError;
 use lib_infra::future::FutureResult;

 pub struct StorageObject {
+  pub workspace_id: String,
  pub file_name: String,
  pub value: ObjectValue,
 }
@ -16,8 +17,9 @@ impl StorageObject {
  /// * `name`: The name of the storage object.
  /// * `file_path`: The file path to the storage object's data.
  ///
-  pub fn from_file<T: ToString>(file_name: &str, file_path: T) -> Self {
+  pub fn from_file<T: ToString>(workspace_id: &str, file_name: &str, file_path: T) -> Self {
    Self {
+      workspace_id: workspace_id.to_string(),
      file_name: file_name.to_string(),
      value: ObjectValue::File {
        file_path: file_path.to_string(),
@ -33,9 +35,15 @@ impl StorageObject {
  /// * `bytes`: The byte data of the storage object.
  /// * `mime`: The MIME type of the storage object.
  ///
-  pub fn from_bytes<B: Into<Bytes>>(file_name: &str, bytes: B, mime: String) -> Self {
+  pub fn from_bytes<B: Into<Bytes>>(
+    workspace_id: &str,
+    file_name: &str,
+    bytes: B,
+    mime: String,
+  ) -> Self {
    let bytes = bytes.into();
    Self {
+      workspace_id: workspace_id.to_string(),
      file_name: file_name.to_string(),
      value: ObjectValue::Bytes { bytes, mime },
    }