refactor: File upload (#5542)

* chore: rename service

* refactor: upload

* chore: save upload meta data

* chore: add sql test

* chore: uploader

* chore: fix upload

* chore: cache file and remove after finish

* chore: retry upload

* chore: pause when netowork unreachable

* chore: add event test

* chore: add test

* chore: clippy

* chore: update client-api commit id

* chore: fix flutter test
This commit is contained in:
Nathan.fooo
2024-06-20 07:44:57 +08:00
committed by GitHub
parent fdaca36b87
commit b64da2c02f
61 changed files with 2687 additions and 643 deletions

View File

@ -31,6 +31,7 @@ collab = { workspace = true }
diesel.workspace = true
uuid.workspace = true
flowy-storage = { workspace = true }
flowy-storage-pub = { workspace = true }
client-api.workspace = true
flowy-chat = { workspace = true }
flowy-chat-pub = { workspace = true }

View File

@ -8,7 +8,7 @@ use flowy_document::entities::{DocumentSnapshotData, DocumentSnapshotMeta};
use flowy_document::manager::{DocumentManager, DocumentSnapshotService, DocumentUserService};
use flowy_document_pub::cloud::DocumentCloudService;
use flowy_error::{FlowyError, FlowyResult};
use flowy_storage::ObjectStorageService;
use flowy_storage_pub::storage::StorageService;
use flowy_user::services::authenticate_user::AuthenticateUser;
pub struct DocumentDepsResolver();
@ -18,7 +18,7 @@ impl DocumentDepsResolver {
_database_manager: &Arc<DatabaseManager>,
collab_builder: Arc<AppFlowyCollabBuilder>,
cloud_service: Arc<dyn DocumentCloudService>,
storage_service: Weak<dyn ObjectStorageService>,
storage_service: Weak<dyn StorageService>,
) -> Arc<DocumentManager> {
let user_service: Arc<dyn DocumentUserService> =
Arc::new(DocumentUserImpl(authenticate_user.clone()));

View File

@ -0,0 +1,54 @@
use flowy_error::FlowyError;
use flowy_sqlite::DBConnection;
use flowy_storage::manager::{StorageManager, StorageUserService};
use flowy_storage_pub::cloud::StorageCloudService;
use flowy_user::services::authenticate_user::AuthenticateUser;
use std::sync::{Arc, Weak};
pub struct FileStorageResolver;
impl FileStorageResolver {
pub fn resolve(
authenticate_user: Weak<AuthenticateUser>,
cloud_service: Arc<dyn StorageCloudService>,
root: &str,
) -> Arc<StorageManager> {
let user_service = FileStorageServiceImpl {
user: authenticate_user,
root_dir: root.to_owned(),
};
Arc::new(StorageManager::new(cloud_service, Arc::new(user_service)))
}
}
struct FileStorageServiceImpl {
user: Weak<AuthenticateUser>,
root_dir: String,
}
impl FileStorageServiceImpl {
fn upgrade_user(&self) -> Result<Arc<AuthenticateUser>, FlowyError> {
let user = self
.user
.upgrade()
.ok_or(FlowyError::internal().with_context("Unexpected error: UserSession is None"))?;
Ok(user)
}
}
impl StorageUserService for FileStorageServiceImpl {
fn user_id(&self) -> Result<i64, FlowyError> {
self.upgrade_user()?.user_id()
}
fn workspace_id(&self) -> Result<String, FlowyError> {
self.upgrade_user()?.workspace_id()
}
fn sqlite_connection(&self, uid: i64) -> Result<DBConnection, FlowyError> {
self.upgrade_user()?.get_sqlite_connection(uid)
}
fn get_application_root_dir(&self) -> &str {
&self.root_dir
}
}

View File

@ -12,5 +12,6 @@ mod folder_deps;
mod chat_deps;
mod database_deps;
pub mod file_storage_deps;
mod search_deps;
mod user_deps;

View File

@ -53,6 +53,7 @@ pub fn create_log_filter(level: String, with_crates: Vec<String>, platform: Plat
filters.push(format!("lib_infra={}", level));
filters.push(format!("flowy_search={}", level));
filters.push(format!("flowy_chat={}", level));
filters.push(format!("flowy_storage={}", level));
// Enable the frontend logs. DO NOT DISABLE.
// These logs are essential for debugging and verifying frontend behavior.
filters.push(format!("dart_ffi={}", level));

View File

@ -1,6 +1,5 @@
use client_api::entity::search_dto::SearchDocumentResponseItem;
use flowy_search_pub::cloud::SearchCloudService;
use flowy_storage::{ObjectIdentity, ObjectStorageService};
use std::sync::Arc;
use anyhow::Error;
@ -28,13 +27,14 @@ use flowy_database_pub::cloud::{
};
use flowy_document::deps::DocumentData;
use flowy_document_pub::cloud::{DocumentCloudService, DocumentSnapshot};
use flowy_error::FlowyError;
use flowy_error::{FlowyError, FlowyResult};
use flowy_folder_pub::cloud::{
FolderCloudService, FolderCollabParams, FolderData, FolderSnapshot, Workspace, WorkspaceRecord,
};
use flowy_server_pub::af_cloud_config::AFCloudConfiguration;
use flowy_server_pub::supabase_config::SupabaseConfiguration;
use flowy_storage::ObjectValue;
use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService};
use flowy_storage_pub::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse};
use flowy_user_pub::cloud::{UserCloudService, UserCloudServiceProvider};
use flowy_user_pub::entities::{Authenticator, UserTokenState};
use lib_infra::async_trait::async_trait;
@ -42,7 +42,8 @@ use lib_infra::future::FutureResult;
use crate::integrate::server::{Server, ServerProvider};
impl ObjectStorageService for ServerProvider {
#[async_trait]
impl StorageCloudService for ServerProvider {
fn get_object_url(&self, object_id: ObjectIdentity) -> FutureResult<String, FlowyError> {
let server = self.get_server();
FutureResult::new(async move {
@ -59,21 +60,85 @@ impl ObjectStorageService for ServerProvider {
})
}
fn delete_object(&self, url: String) -> FutureResult<(), FlowyError> {
fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError> {
let server = self.get_server();
let url = url.to_string();
FutureResult::new(async move {
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
storage.delete_object(url).await
storage.delete_object(&url).await
})
}
fn get_object(&self, url: String) -> FutureResult<flowy_storage::ObjectValue, FlowyError> {
fn get_object(&self, url: String) -> FutureResult<ObjectValue, FlowyError> {
let server = self.get_server();
FutureResult::new(async move {
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
storage.get_object(url).await
})
}
fn get_object_url_v1(
&self,
workspace_id: &str,
parent_dir: &str,
file_id: &str,
) -> FlowyResult<String> {
let server = self.get_server()?;
let storage = server.file_storage().ok_or(FlowyError::internal())?;
storage.get_object_url_v1(workspace_id, parent_dir, file_id)
}
async fn create_upload(
&self,
workspace_id: &str,
parent_dir: &str,
file_id: &str,
content_type: &str,
) -> Result<CreateUploadResponse, FlowyError> {
let server = self.get_server();
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
storage
.create_upload(workspace_id, parent_dir, file_id, content_type)
.await
}
async fn upload_part(
&self,
workspace_id: &str,
parent_dir: &str,
upload_id: &str,
file_id: &str,
part_number: i32,
body: Vec<u8>,
) -> Result<UploadPartResponse, FlowyError> {
let server = self.get_server();
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
storage
.upload_part(
workspace_id,
parent_dir,
upload_id,
file_id,
part_number,
body,
)
.await
}
async fn complete_upload(
&self,
workspace_id: &str,
parent_dir: &str,
upload_id: &str,
file_id: &str,
parts: Vec<CompletedPartRequest>,
) -> Result<(), FlowyError> {
let server = self.get_server();
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
storage
.complete_upload(workspace_id, parent_dir, upload_id, file_id, parts)
.await
}
}
impl UserCloudServiceProvider for ServerProvider {

View File

@ -1,7 +1,7 @@
use std::sync::Arc;
use anyhow::Context;
use tracing::event;
use tracing::{event, trace};
use collab_entity::CollabType;
use collab_integrate::collab_builder::AppFlowyCollabBuilder;
@ -9,13 +9,13 @@ use flowy_database2::DatabaseManager;
use flowy_document::manager::DocumentManager;
use flowy_error::{FlowyError, FlowyResult};
use flowy_folder::manager::{FolderInitDataSource, FolderManager};
use flowy_storage::manager::StorageManager;
use flowy_user::event_map::UserStatusCallback;
use flowy_user_pub::cloud::{UserCloudConfig, UserCloudServiceProvider};
use flowy_user_pub::entities::{Authenticator, UserProfile, UserWorkspace};
use lib_infra::future::{to_fut, Fut};
use crate::integrate::server::{Server, ServerProvider};
use crate::AppFlowyCoreConfig;
pub(crate) struct UserStatusCallbackImpl {
pub(crate) collab_builder: Arc<AppFlowyCollabBuilder>,
@ -23,8 +23,7 @@ pub(crate) struct UserStatusCallbackImpl {
pub(crate) database_manager: Arc<DatabaseManager>,
pub(crate) document_manager: Arc<DocumentManager>,
pub(crate) server_provider: Arc<ServerProvider>,
#[allow(dead_code)]
pub(crate) config: AppFlowyCoreConfig,
pub(crate) storage_manager: Arc<StorageManager>,
}
impl UserStatusCallback for UserStatusCallbackImpl {
@ -213,6 +212,8 @@ impl UserStatusCallback for UserStatusCallbackImpl {
}
fn did_update_network(&self, reachable: bool) {
trace!("Notify did update network: reachable: {}", reachable);
self.collab_builder.update_network(reachable);
self.storage_manager.update_network_reachable(reachable);
}
}

View File

@ -2,7 +2,6 @@
use flowy_search::folder::indexer::FolderIndexManagerImpl;
use flowy_search::services::manager::SearchManager;
use flowy_storage::ObjectStorageService;
use std::sync::{Arc, Weak};
use std::time::Duration;
use sysinfo::System;
@ -18,6 +17,7 @@ use flowy_folder::manager::FolderManager;
use flowy_server::af_cloud::define::ServerUser;
use flowy_sqlite::kv::StorePreferences;
use flowy_storage::manager::StorageManager;
use flowy_user::services::authenticate_user::AuthenticateUser;
use flowy_user::services::entities::UserConfig;
use flowy_user::user_manager::UserManager;
@ -30,6 +30,7 @@ use lib_log::stream_log::StreamLogSender;
use module::make_plugins;
use crate::config::AppFlowyCoreConfig;
use crate::deps_resolve::file_storage_deps::FileStorageResolver;
use crate::deps_resolve::*;
use crate::integrate::collab_interact::CollabInteractImpl;
use crate::integrate::log::init_log;
@ -59,6 +60,7 @@ pub struct AppFlowyCore {
pub store_preference: Arc<StorePreferences>,
pub search_manager: Arc<SearchManager>,
pub chat_manager: Arc<ChatManager>,
pub storage_manager: Arc<StorageManager>,
}
impl AppFlowyCore {
@ -140,7 +142,13 @@ impl AppFlowyCore {
collab_builder,
search_manager,
chat_manager,
storage_manager,
) = async {
let storage_manager = FileStorageResolver::resolve(
Arc::downgrade(&authenticate_user),
server_provider.clone(),
&user_config.storage_path,
);
/// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded
/// on demand based on the [CollabPluginConfig].
let collab_builder = Arc::new(AppFlowyCollabBuilder::new(
@ -164,7 +172,7 @@ impl AppFlowyCore {
&database_manager,
collab_builder.clone(),
server_provider.clone(),
Arc::downgrade(&(server_provider.clone() as Arc<dyn ObjectStorageService>)),
Arc::downgrade(&storage_manager.storage_service),
);
let chat_manager =
@ -216,6 +224,7 @@ impl AppFlowyCore {
collab_builder,
search_manager,
chat_manager,
storage_manager,
)
}
.await;
@ -226,7 +235,7 @@ impl AppFlowyCore {
database_manager: database_manager.clone(),
document_manager: document_manager.clone(),
server_provider: server_provider.clone(),
config: config.clone(),
storage_manager: storage_manager.clone(),
};
let collab_interact_impl = CollabInteractImpl {
@ -267,6 +276,7 @@ impl AppFlowyCore {
store_preference,
search_manager,
chat_manager,
storage_manager,
}
}