mirror of
https://github.com/AppFlowy-IO/AppFlowy.git
synced 2024-08-30 18:12:39 +00:00
refactor: File upload (#5542)
* chore: rename service * refactor: upload * chore: save upload meta data * chore: add sql test * chore: uploader * chore: fix upload * chore: cache file and remove after finish * chore: retry upload * chore: pause when netowork unreachable * chore: add event test * chore: add test * chore: clippy * chore: update client-api commit id * chore: fix flutter test
This commit is contained in:
@ -31,6 +31,7 @@ collab = { workspace = true }
|
||||
diesel.workspace = true
|
||||
uuid.workspace = true
|
||||
flowy-storage = { workspace = true }
|
||||
flowy-storage-pub = { workspace = true }
|
||||
client-api.workspace = true
|
||||
flowy-chat = { workspace = true }
|
||||
flowy-chat-pub = { workspace = true }
|
||||
|
@ -8,7 +8,7 @@ use flowy_document::entities::{DocumentSnapshotData, DocumentSnapshotMeta};
|
||||
use flowy_document::manager::{DocumentManager, DocumentSnapshotService, DocumentUserService};
|
||||
use flowy_document_pub::cloud::DocumentCloudService;
|
||||
use flowy_error::{FlowyError, FlowyResult};
|
||||
use flowy_storage::ObjectStorageService;
|
||||
use flowy_storage_pub::storage::StorageService;
|
||||
use flowy_user::services::authenticate_user::AuthenticateUser;
|
||||
|
||||
pub struct DocumentDepsResolver();
|
||||
@ -18,7 +18,7 @@ impl DocumentDepsResolver {
|
||||
_database_manager: &Arc<DatabaseManager>,
|
||||
collab_builder: Arc<AppFlowyCollabBuilder>,
|
||||
cloud_service: Arc<dyn DocumentCloudService>,
|
||||
storage_service: Weak<dyn ObjectStorageService>,
|
||||
storage_service: Weak<dyn StorageService>,
|
||||
) -> Arc<DocumentManager> {
|
||||
let user_service: Arc<dyn DocumentUserService> =
|
||||
Arc::new(DocumentUserImpl(authenticate_user.clone()));
|
||||
|
@ -0,0 +1,54 @@
|
||||
use flowy_error::FlowyError;
|
||||
use flowy_sqlite::DBConnection;
|
||||
use flowy_storage::manager::{StorageManager, StorageUserService};
|
||||
use flowy_storage_pub::cloud::StorageCloudService;
|
||||
use flowy_user::services::authenticate_user::AuthenticateUser;
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
pub struct FileStorageResolver;
|
||||
|
||||
impl FileStorageResolver {
|
||||
pub fn resolve(
|
||||
authenticate_user: Weak<AuthenticateUser>,
|
||||
cloud_service: Arc<dyn StorageCloudService>,
|
||||
root: &str,
|
||||
) -> Arc<StorageManager> {
|
||||
let user_service = FileStorageServiceImpl {
|
||||
user: authenticate_user,
|
||||
root_dir: root.to_owned(),
|
||||
};
|
||||
Arc::new(StorageManager::new(cloud_service, Arc::new(user_service)))
|
||||
}
|
||||
}
|
||||
|
||||
struct FileStorageServiceImpl {
|
||||
user: Weak<AuthenticateUser>,
|
||||
root_dir: String,
|
||||
}
|
||||
impl FileStorageServiceImpl {
|
||||
fn upgrade_user(&self) -> Result<Arc<AuthenticateUser>, FlowyError> {
|
||||
let user = self
|
||||
.user
|
||||
.upgrade()
|
||||
.ok_or(FlowyError::internal().with_context("Unexpected error: UserSession is None"))?;
|
||||
Ok(user)
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageUserService for FileStorageServiceImpl {
|
||||
fn user_id(&self) -> Result<i64, FlowyError> {
|
||||
self.upgrade_user()?.user_id()
|
||||
}
|
||||
|
||||
fn workspace_id(&self) -> Result<String, FlowyError> {
|
||||
self.upgrade_user()?.workspace_id()
|
||||
}
|
||||
|
||||
fn sqlite_connection(&self, uid: i64) -> Result<DBConnection, FlowyError> {
|
||||
self.upgrade_user()?.get_sqlite_connection(uid)
|
||||
}
|
||||
|
||||
fn get_application_root_dir(&self) -> &str {
|
||||
&self.root_dir
|
||||
}
|
||||
}
|
@ -12,5 +12,6 @@ mod folder_deps;
|
||||
|
||||
mod chat_deps;
|
||||
mod database_deps;
|
||||
pub mod file_storage_deps;
|
||||
mod search_deps;
|
||||
mod user_deps;
|
||||
|
@ -53,6 +53,7 @@ pub fn create_log_filter(level: String, with_crates: Vec<String>, platform: Plat
|
||||
filters.push(format!("lib_infra={}", level));
|
||||
filters.push(format!("flowy_search={}", level));
|
||||
filters.push(format!("flowy_chat={}", level));
|
||||
filters.push(format!("flowy_storage={}", level));
|
||||
// Enable the frontend logs. DO NOT DISABLE.
|
||||
// These logs are essential for debugging and verifying frontend behavior.
|
||||
filters.push(format!("dart_ffi={}", level));
|
||||
|
@ -1,6 +1,5 @@
|
||||
use client_api::entity::search_dto::SearchDocumentResponseItem;
|
||||
use flowy_search_pub::cloud::SearchCloudService;
|
||||
use flowy_storage::{ObjectIdentity, ObjectStorageService};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Error;
|
||||
@ -28,13 +27,14 @@ use flowy_database_pub::cloud::{
|
||||
};
|
||||
use flowy_document::deps::DocumentData;
|
||||
use flowy_document_pub::cloud::{DocumentCloudService, DocumentSnapshot};
|
||||
use flowy_error::FlowyError;
|
||||
use flowy_error::{FlowyError, FlowyResult};
|
||||
use flowy_folder_pub::cloud::{
|
||||
FolderCloudService, FolderCollabParams, FolderData, FolderSnapshot, Workspace, WorkspaceRecord,
|
||||
};
|
||||
use flowy_server_pub::af_cloud_config::AFCloudConfiguration;
|
||||
use flowy_server_pub::supabase_config::SupabaseConfiguration;
|
||||
use flowy_storage::ObjectValue;
|
||||
use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService};
|
||||
use flowy_storage_pub::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse};
|
||||
use flowy_user_pub::cloud::{UserCloudService, UserCloudServiceProvider};
|
||||
use flowy_user_pub::entities::{Authenticator, UserTokenState};
|
||||
use lib_infra::async_trait::async_trait;
|
||||
@ -42,7 +42,8 @@ use lib_infra::future::FutureResult;
|
||||
|
||||
use crate::integrate::server::{Server, ServerProvider};
|
||||
|
||||
impl ObjectStorageService for ServerProvider {
|
||||
#[async_trait]
|
||||
impl StorageCloudService for ServerProvider {
|
||||
fn get_object_url(&self, object_id: ObjectIdentity) -> FutureResult<String, FlowyError> {
|
||||
let server = self.get_server();
|
||||
FutureResult::new(async move {
|
||||
@ -59,21 +60,85 @@ impl ObjectStorageService for ServerProvider {
|
||||
})
|
||||
}
|
||||
|
||||
fn delete_object(&self, url: String) -> FutureResult<(), FlowyError> {
|
||||
fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError> {
|
||||
let server = self.get_server();
|
||||
let url = url.to_string();
|
||||
FutureResult::new(async move {
|
||||
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage.delete_object(url).await
|
||||
storage.delete_object(&url).await
|
||||
})
|
||||
}
|
||||
|
||||
fn get_object(&self, url: String) -> FutureResult<flowy_storage::ObjectValue, FlowyError> {
|
||||
fn get_object(&self, url: String) -> FutureResult<ObjectValue, FlowyError> {
|
||||
let server = self.get_server();
|
||||
FutureResult::new(async move {
|
||||
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage.get_object(url).await
|
||||
})
|
||||
}
|
||||
|
||||
fn get_object_url_v1(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
parent_dir: &str,
|
||||
file_id: &str,
|
||||
) -> FlowyResult<String> {
|
||||
let server = self.get_server()?;
|
||||
let storage = server.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage.get_object_url_v1(workspace_id, parent_dir, file_id)
|
||||
}
|
||||
|
||||
async fn create_upload(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
parent_dir: &str,
|
||||
file_id: &str,
|
||||
content_type: &str,
|
||||
) -> Result<CreateUploadResponse, FlowyError> {
|
||||
let server = self.get_server();
|
||||
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage
|
||||
.create_upload(workspace_id, parent_dir, file_id, content_type)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upload_part(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
parent_dir: &str,
|
||||
upload_id: &str,
|
||||
file_id: &str,
|
||||
part_number: i32,
|
||||
body: Vec<u8>,
|
||||
) -> Result<UploadPartResponse, FlowyError> {
|
||||
let server = self.get_server();
|
||||
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage
|
||||
.upload_part(
|
||||
workspace_id,
|
||||
parent_dir,
|
||||
upload_id,
|
||||
file_id,
|
||||
part_number,
|
||||
body,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn complete_upload(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
parent_dir: &str,
|
||||
upload_id: &str,
|
||||
file_id: &str,
|
||||
parts: Vec<CompletedPartRequest>,
|
||||
) -> Result<(), FlowyError> {
|
||||
let server = self.get_server();
|
||||
let storage = server?.file_storage().ok_or(FlowyError::internal())?;
|
||||
storage
|
||||
.complete_upload(workspace_id, parent_dir, upload_id, file_id, parts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl UserCloudServiceProvider for ServerProvider {
|
||||
|
@ -1,7 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Context;
|
||||
use tracing::event;
|
||||
use tracing::{event, trace};
|
||||
|
||||
use collab_entity::CollabType;
|
||||
use collab_integrate::collab_builder::AppFlowyCollabBuilder;
|
||||
@ -9,13 +9,13 @@ use flowy_database2::DatabaseManager;
|
||||
use flowy_document::manager::DocumentManager;
|
||||
use flowy_error::{FlowyError, FlowyResult};
|
||||
use flowy_folder::manager::{FolderInitDataSource, FolderManager};
|
||||
use flowy_storage::manager::StorageManager;
|
||||
use flowy_user::event_map::UserStatusCallback;
|
||||
use flowy_user_pub::cloud::{UserCloudConfig, UserCloudServiceProvider};
|
||||
use flowy_user_pub::entities::{Authenticator, UserProfile, UserWorkspace};
|
||||
use lib_infra::future::{to_fut, Fut};
|
||||
|
||||
use crate::integrate::server::{Server, ServerProvider};
|
||||
use crate::AppFlowyCoreConfig;
|
||||
|
||||
pub(crate) struct UserStatusCallbackImpl {
|
||||
pub(crate) collab_builder: Arc<AppFlowyCollabBuilder>,
|
||||
@ -23,8 +23,7 @@ pub(crate) struct UserStatusCallbackImpl {
|
||||
pub(crate) database_manager: Arc<DatabaseManager>,
|
||||
pub(crate) document_manager: Arc<DocumentManager>,
|
||||
pub(crate) server_provider: Arc<ServerProvider>,
|
||||
#[allow(dead_code)]
|
||||
pub(crate) config: AppFlowyCoreConfig,
|
||||
pub(crate) storage_manager: Arc<StorageManager>,
|
||||
}
|
||||
|
||||
impl UserStatusCallback for UserStatusCallbackImpl {
|
||||
@ -213,6 +212,8 @@ impl UserStatusCallback for UserStatusCallbackImpl {
|
||||
}
|
||||
|
||||
fn did_update_network(&self, reachable: bool) {
|
||||
trace!("Notify did update network: reachable: {}", reachable);
|
||||
self.collab_builder.update_network(reachable);
|
||||
self.storage_manager.update_network_reachable(reachable);
|
||||
}
|
||||
}
|
||||
|
@ -2,7 +2,6 @@
|
||||
|
||||
use flowy_search::folder::indexer::FolderIndexManagerImpl;
|
||||
use flowy_search::services::manager::SearchManager;
|
||||
use flowy_storage::ObjectStorageService;
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::time::Duration;
|
||||
use sysinfo::System;
|
||||
@ -18,6 +17,7 @@ use flowy_folder::manager::FolderManager;
|
||||
use flowy_server::af_cloud::define::ServerUser;
|
||||
|
||||
use flowy_sqlite::kv::StorePreferences;
|
||||
use flowy_storage::manager::StorageManager;
|
||||
use flowy_user::services::authenticate_user::AuthenticateUser;
|
||||
use flowy_user::services::entities::UserConfig;
|
||||
use flowy_user::user_manager::UserManager;
|
||||
@ -30,6 +30,7 @@ use lib_log::stream_log::StreamLogSender;
|
||||
use module::make_plugins;
|
||||
|
||||
use crate::config::AppFlowyCoreConfig;
|
||||
use crate::deps_resolve::file_storage_deps::FileStorageResolver;
|
||||
use crate::deps_resolve::*;
|
||||
use crate::integrate::collab_interact::CollabInteractImpl;
|
||||
use crate::integrate::log::init_log;
|
||||
@ -59,6 +60,7 @@ pub struct AppFlowyCore {
|
||||
pub store_preference: Arc<StorePreferences>,
|
||||
pub search_manager: Arc<SearchManager>,
|
||||
pub chat_manager: Arc<ChatManager>,
|
||||
pub storage_manager: Arc<StorageManager>,
|
||||
}
|
||||
|
||||
impl AppFlowyCore {
|
||||
@ -140,7 +142,13 @@ impl AppFlowyCore {
|
||||
collab_builder,
|
||||
search_manager,
|
||||
chat_manager,
|
||||
storage_manager,
|
||||
) = async {
|
||||
let storage_manager = FileStorageResolver::resolve(
|
||||
Arc::downgrade(&authenticate_user),
|
||||
server_provider.clone(),
|
||||
&user_config.storage_path,
|
||||
);
|
||||
/// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded
|
||||
/// on demand based on the [CollabPluginConfig].
|
||||
let collab_builder = Arc::new(AppFlowyCollabBuilder::new(
|
||||
@ -164,7 +172,7 @@ impl AppFlowyCore {
|
||||
&database_manager,
|
||||
collab_builder.clone(),
|
||||
server_provider.clone(),
|
||||
Arc::downgrade(&(server_provider.clone() as Arc<dyn ObjectStorageService>)),
|
||||
Arc::downgrade(&storage_manager.storage_service),
|
||||
);
|
||||
|
||||
let chat_manager =
|
||||
@ -216,6 +224,7 @@ impl AppFlowyCore {
|
||||
collab_builder,
|
||||
search_manager,
|
||||
chat_manager,
|
||||
storage_manager,
|
||||
)
|
||||
}
|
||||
.await;
|
||||
@ -226,7 +235,7 @@ impl AppFlowyCore {
|
||||
database_manager: database_manager.clone(),
|
||||
document_manager: document_manager.clone(),
|
||||
server_provider: server_provider.clone(),
|
||||
config: config.clone(),
|
||||
storage_manager: storage_manager.clone(),
|
||||
};
|
||||
|
||||
let collab_interact_impl = CollabInteractImpl {
|
||||
@ -267,6 +276,7 @@ impl AppFlowyCore {
|
||||
store_preference,
|
||||
search_manager,
|
||||
chat_manager,
|
||||
storage_manager,
|
||||
}
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user