refactor: File upload (#5542)

* chore: rename service

* refactor: upload

* chore: save upload meta data

* chore: add sql test

* chore: uploader

* chore: fix upload

* chore: cache file and remove after finish

* chore: retry upload

* chore: pause when netowork unreachable

* chore: add event test

* chore: add test

* chore: clippy

* chore: update client-api commit id

* chore: fix flutter test

frontend/rust-lib/flowy-document/Cargo.toml

@@ -14,7 +14,7 @@ collab-entity = { workspace = true }
 collab-plugins = { workspace = true }
 collab-integrate = { workspace = true }
 flowy-document-pub = { workspace = true }
-flowy-storage = { workspace = true }
+flowy-storage-pub = { workspace = true }
 flowy-derive.workspace = true
 flowy-notification = { workspace = true }
 flowy-error = { path = "../flowy-error", features = ["impl_from_serde", "impl_from_dispatch_error", "impl_from_collab_document", "impl_from_collab_persistence"] }

frontend/rust-lib/flowy-document/src/entities.rs

@@ -77,11 +77,12 @@ pub struct UploadFileParamsPB {
   pub workspace_id: String,
 
   #[pb(index = 2)]
-  #[validate(custom = "required_valid_path")]
-  pub local_file_path: String,
+  #[validate(custom = "required_not_empty_str")]
+  pub document_id: String,
 
   #[pb(index = 3)]
-  pub is_async: bool,
+  #[validate(custom = "required_valid_path")]
+  pub local_file_path: String,
 }
 
 #[derive(Default, ProtoBuf, Validate)]

frontend/rust-lib/flowy-document/src/event_handler.rs

@@ -422,13 +422,13 @@ pub(crate) async fn upload_file_handler(
 ) -> DataResult<UploadedFilePB, FlowyError> {
   let AFPluginData(UploadFileParamsPB {
     workspace_id,
+    document_id,
     local_file_path,
-    is_async,
   }) = params;
 
   let manager = upgrade_document(manager)?;
   let url = manager
-    .upload_file(workspace_id, &local_file_path, is_async)
+    .upload_file(workspace_id, &document_id, &local_file_path)
     .await?;
 
   Ok(AFPluginData(UploadedFilePB {

frontend/rust-lib/flowy-document/src/manager.rs

@@ -13,16 +13,14 @@ use collab_document::document_data::default_document_data;
 use collab_entity::CollabType;
 use collab_plugins::CollabKVDB;
 use dashmap::DashMap;
-use flowy_storage::object_from_disk;
 use lib_infra::util::timestamp;
-use tokio::io::AsyncWriteExt;
-use tracing::{error, trace};
+use tracing::trace;
 use tracing::{event, instrument};
 
 use collab_integrate::collab_builder::{AppFlowyCollabBuilder, CollabBuilderConfig};
 use flowy_document_pub::cloud::DocumentCloudService;
 use flowy_error::{internal_error, ErrorCode, FlowyError, FlowyResult};
-use flowy_storage::ObjectStorageService;
+use flowy_storage_pub::storage::StorageService;
 use lib_dispatch::prelude::af_spawn;
 
 use crate::document::MutexDocument;
@@ -53,7 +51,7 @@ pub struct DocumentManager {
   documents: Arc<DashMap<String, Arc<MutexDocument>>>,
   removing_documents: Arc<DashMap<String, Arc<MutexDocument>>>,
   cloud_service: Arc<dyn DocumentCloudService>,
-  storage_service: Weak<dyn ObjectStorageService>,
+  storage_service: Weak<dyn StorageService>,
   snapshot_service: Arc<dyn DocumentSnapshotService>,
 }
 
@@ -62,7 +60,7 @@ impl DocumentManager {
     user_service: Arc<dyn DocumentUserService>,
     collab_builder: Arc<AppFlowyCollabBuilder>,
     cloud_service: Arc<dyn DocumentCloudService>,
-    storage_service: Weak<dyn ObjectStorageService>,
+    storage_service: Weak<dyn StorageService>,
     snapshot_service: Arc<dyn DocumentSnapshotService>,
   ) -> Self {
     Self {
@@ -323,73 +321,30 @@ impl DocumentManager {
     Ok(snapshot)
   }
 
+  #[instrument(level = "debug", skip_all, err)]
   pub async fn upload_file(
     &self,
     workspace_id: String,
+    document_id: &str,
     local_file_path: &str,
-    is_async: bool,
   ) -> FlowyResult<String> {
-    let (object_identity, object_value) = object_from_disk(&workspace_id, local_file_path).await?;
     let storage_service = self.storage_service_upgrade()?;
-    let url = storage_service.get_object_url(object_identity).await?;
-
-    let clone_url = url.clone();
-
-    match is_async {
-      false => storage_service.put_object(clone_url, object_value).await?,
-      true => {
-        // let the upload happen in the background
-        af_spawn(async move {
-          if let Err(e) = storage_service.put_object(clone_url, object_value).await {
-            error!("upload file failed: {}", e);
-          }
-        });
-      },
-    }
+    let url = storage_service
+      .create_upload(&workspace_id, document_id, local_file_path)
+      .await?
+      .url;
     Ok(url)
   }
 
   pub async fn download_file(&self, local_file_path: String, url: String) -> FlowyResult<()> {
-    // TODO(nathan): save file when the current target is wasm
-    #[cfg(not(target_arch = "wasm32"))]
-    {
-      if tokio::fs::metadata(&local_file_path).await.is_ok() {
-        tracing::warn!("file already exist in user local disk: {}", local_file_path);
-        return Ok(());
-      }
-
-      let storage_service = self.storage_service_upgrade()?;
-      let object_value = storage_service.get_object(url).await?;
-      // create file if not exist
-      let mut file = tokio::fs::OpenOptions::new()
-        .create(true)
-        .truncate(true)
-        .write(true)
-        .open(&local_file_path)
-        .await?;
-
-      let n = file.write(&object_value.raw).await?;
-      tracing::info!("downloaded {} bytes to file: {}", n, local_file_path);
-    }
+    let storage_service = self.storage_service_upgrade()?;
+    storage_service.download_object(url, local_file_path)?;
     Ok(())
   }
 
   pub async fn delete_file(&self, local_file_path: String, url: String) -> FlowyResult<()> {
-    // TODO(nathan): delete file when the current target is wasm
-    #[cfg(not(target_arch = "wasm32"))]
-    // delete file from local
-    tokio::fs::remove_file(local_file_path).await?;
-
-    // delete from cloud
     let storage_service = self.storage_service_upgrade()?;
-    af_spawn(async move {
-      if let Err(e) = storage_service.delete_object(url).await {
-        // TODO: add WAL to log the delete operation.
-        // keep a list of files to be deleted, and retry later
-        error!("delete file failed: {}", e);
-      }
-    });
-
+    storage_service.delete_object(url, local_file_path)?;
     Ok(())
   }
 
@@ -424,7 +379,7 @@ impl DocumentManager {
     }
   }
 
-  fn storage_service_upgrade(&self) -> FlowyResult<Arc<dyn ObjectStorageService>> {
+  fn storage_service_upgrade(&self) -> FlowyResult<Arc<dyn StorageService>> {
     let storage_service = self.storage_service.upgrade().ok_or_else(|| {
       FlowyError::internal().with_context("The file storage service is already dropped")
     })?;
@@ -438,7 +393,7 @@ impl DocumentManager {
   }
   /// Only expose this method for testing
   #[cfg(debug_assertions)]
-  pub fn get_file_storage_service(&self) -> &Weak<dyn ObjectStorageService> {
+  pub fn get_file_storage_service(&self) -> &Weak<dyn StorageService> {
     &self.storage_service
   }
 

frontend/rust-lib/flowy-document/tests/document/util.rs

@@ -20,8 +20,10 @@ use flowy_document::entities::{DocumentSnapshotData, DocumentSnapshotMeta};
 use flowy_document::manager::{DocumentManager, DocumentSnapshotService, DocumentUserService};
 use flowy_document_pub::cloud::*;
 use flowy_error::{ErrorCode, FlowyError, FlowyResult};
-use flowy_storage::ObjectStorageService;
+use flowy_storage_pub::chunked_byte::ChunkedBytes;
+use flowy_storage_pub::storage::{CreatedUpload, StorageService};
 use lib_infra::async_trait::async_trait;
+use lib_infra::box_any::BoxAny;
 use lib_infra::future::FutureResult;
 
 pub struct DocumentTest {
@@ -32,7 +34,7 @@ impl DocumentTest {
   pub fn new() -> Self {
     let user = FakeUser::new();
     let cloud_service = Arc::new(LocalTestDocumentCloudServiceImpl());
-    let file_storage = Arc::new(DocumentTestFileStorageService) as Arc<dyn ObjectStorageService>;
+    let file_storage = Arc::new(DocumentTestFileStorageService) as Arc<dyn StorageService>;
     let document_snapshot = Arc::new(DocumentTestSnapshot);
 
     let builder = Arc::new(AppFlowyCollabBuilder::new(
@@ -173,27 +175,44 @@ impl DocumentCloudService for LocalTestDocumentCloudServiceImpl {
 }
 
 pub struct DocumentTestFileStorageService;
-impl ObjectStorageService for DocumentTestFileStorageService {
-  fn get_object_url(
+
+#[async_trait]
+impl StorageService for DocumentTestFileStorageService {
+  fn upload_object(
     &self,
-    _object_id: flowy_storage::ObjectIdentity,
+    _workspace_id: &str,
+    _local_file_path: &str,
   ) -> FutureResult<String, FlowyError> {
     todo!()
   }
 
-  fn put_object(
+  fn delete_object(&self, _url: String, _local_file_path: String) -> FlowyResult<()> {
+    todo!()
+  }
+
+  fn download_object(&self, _url: String, _local_file_path: String) -> FlowyResult<()> {
+    todo!()
+  }
+
+  fn create_upload(
     &self,
-    _url: String,
-    _object_value: flowy_storage::ObjectValue,
-  ) -> FutureResult<(), FlowyError> {
+    _workspace_id: &str,
+    _parent_dir: &str,
+    _local_file_path: &str,
+  ) -> FutureResult<CreatedUpload, flowy_error::FlowyError> {
     todo!()
   }
 
-  fn delete_object(&self, _url: String) -> FutureResult<(), FlowyError> {
+  async fn start_upload(&self, _chunks: &ChunkedBytes, _record: &BoxAny) -> Result<(), FlowyError> {
     todo!()
   }
 
-  fn get_object(&self, _url: String) -> FutureResult<flowy_storage::ObjectValue, FlowyError> {
+  async fn resume_upload(
+    &self,
+    _workspace_id: &str,
+    _parent_dir: &str,
+    _file_id: &str,
+  ) -> Result<(), FlowyError> {
     todo!()
   }
 }