Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: File upload (#5542)

Browse Source 
* chore: rename service

* refactor: upload

* chore: save upload meta data

* chore: add sql test

* chore: uploader

* chore: fix upload

* chore: cache file and remove after finish

* chore: retry upload

* chore: pause when netowork unreachable

* chore: add event test

* chore: add test

* chore: clippy

* chore: update client-api commit id

* chore: fix flutter test
This commit is contained in:
Nathan.fooo
2024-06-20 07:44:57 +08:00
committed by GitHub
parent fdaca36b87
commit b64da2c02f
61 changed files with 2687 additions and 643 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
frontend/rust-lib/flowy-server/Cargo.toml
View File

@ -42,6 +42,7 @@ flowy-server-pub = { workspace = true }
flowy-search-pub = { workspace = true }
flowy-encrypt = { workspace = true }
flowy-storage = { workspace = true }
flowy-storage-pub = { workspace = true }
flowy-chat-pub = { workspace = true }
mime_guess = "2.0"
url = "2.4"

93
frontend/rust-lib/flowy-server/src/af_cloud/impls/file_storage.rs
View File

@ -1,5 +1,8 @@
use flowy_error::FlowyError;
use flowy_storage::{ObjectIdentity, ObjectStorageService, ObjectValue};
use client_api::entity::{CompleteUploadRequest, CreateUploadRequest};
use flowy_error::{FlowyError, FlowyResult};
use flowy_storage_pub::cloud::{ObjectIdentity, ObjectValue, StorageCloudService};
use flowy_storage_pub::storage::{CompletedPartRequest, CreateUploadResponse, UploadPartResponse};
use lib_infra::async_trait::async_trait;
use lib_infra::future::FutureResult;
use crate::af_cloud::AFServer;
@ -12,7 +15,8 @@ impl<T> AFCloudFileStorageServiceImpl<T> {
}
}
impl<T> ObjectStorageService for AFCloudFileStorageServiceImpl<T>
#[async_trait]
impl<T> StorageCloudService for AFCloudFileStorageServiceImpl<T>
where
T: AFServer,
{
@ -36,7 +40,8 @@ where
})
}
fn delete_object(&self, url: String) -> FutureResult<(), FlowyError> {
fn delete_object(&self, url: &str) -> FutureResult<(), FlowyError> {
let url = url.to_string();
let try_get_client = self.0.try_get_client();
FutureResult::new(async move {
let client = try_get_client?;
@ -56,4 +61,84 @@ where
})
})
}
fn get_object_url_v1(
&self,
workspace_id: &str,
parent_dir: &str,
file_id: &str,
) -> FlowyResult<String> {
let client = self.0.try_get_client()?;
let url = client.get_blob_url_v1(workspace_id, parent_dir, file_id);
Ok(url)
}
async fn create_upload(
&self,
workspace_id: &str,
parent_dir: &str,
file_id: &str,
content_type: &str,
) -> Result<CreateUploadResponse, FlowyError> {
let parent_dir = parent_dir.to_string();
let content_type = content_type.to_string();
let file_id = file_id.to_string();
let try_get_client = self.0.try_get_client();
let client = try_get_client?;
let req = CreateUploadRequest {
file_id,
parent_dir,
content_type,
};
let resp = client.create_upload(workspace_id, req).await?;
Ok(resp)
}
async fn upload_part(
&self,
workspace_id: &str,
parent_dir: &str,
upload_id: &str,
file_id: &str,
part_number: i32,
body: Vec<u8>,
) -> Result<UploadPartResponse, FlowyError> {
let try_get_client = self.0.try_get_client();
let client = try_get_client?;
let resp = client
.upload_part(
workspace_id,
parent_dir,
file_id,
upload_id,
part_number,
body,
)
.await?;
Ok(resp)
}
async fn complete_upload(
&self,
workspace_id: &str,
parent_dir: &str,
upload_id: &str,
file_id: &str,
parts: Vec<CompletedPartRequest>,
) -> Result<(), FlowyError> {
let parent_dir = parent_dir.to_string();
let upload_id = upload_id.to_string();
let file_id = file_id.to_string();
let try_get_client = self.0.try_get_client();
let client = try_get_client?;
let request = CompleteUploadRequest {
file_id,
parent_dir,
upload_id,
parts,
};
client.complete_upload(workspace_id, request).await?;
Ok(())
}
}

4
frontend/rust-lib/flowy-server/src/af_cloud/server.rs
View File

@ -12,7 +12,6 @@ use client_api::ws::{
use client_api::{Client, ClientConfiguration};
use flowy_chat_pub::cloud::ChatCloudService;
use flowy_search_pub::cloud::SearchCloudService;
use flowy_storage::ObjectStorageService;
use rand::Rng;
use semver::Version;
use tokio::select;
@ -28,6 +27,7 @@ use flowy_document_pub::cloud::DocumentCloudService;
use flowy_error::{ErrorCode, FlowyError};
use flowy_folder_pub::cloud::FolderCloudService;
use flowy_server_pub::af_cloud_config::AFCloudConfiguration;
use flowy_storage_pub::cloud::StorageCloudService;
use flowy_user_pub::cloud::{UserCloudService, UserUpdate};
use flowy_user_pub::entities::UserTokenState;
use lib_dispatch::prelude::af_spawn;
@ -252,7 +252,7 @@ impl AppFlowyServer for AppFlowyCloudServer {
Ok(channel.map(|c| (c, connect_state_recv, self.ws_client.is_connected())))
}
fn file_storage(&self) -> Option<Arc<dyn ObjectStorageService>> {
fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>> {
let client = AFServerImpl {
client: self.get_client(),
};

4
frontend/rust-lib/flowy-server/src/local_server/server.rs
View File

@ -1,5 +1,4 @@
use flowy_search_pub::cloud::SearchCloudService;
use flowy_storage::ObjectStorageService;
use std::sync::Arc;
use parking_lot::RwLock;
@ -9,6 +8,7 @@ use flowy_database_pub::cloud::DatabaseCloudService;
use flowy_document_pub::cloud::DocumentCloudService;
use flowy_error::FlowyError;
use flowy_folder_pub::cloud::FolderCloudService;
use flowy_storage_pub::cloud::StorageCloudService;
// use flowy_user::services::database::{
// get_user_profile, get_user_workspace, open_collab_db, open_user_db,
// };
@ -68,7 +68,7 @@ impl AppFlowyServer for LocalServer {
Arc::new(LocalServerDocumentCloudServiceImpl())
}
fn file_storage(&self) -> Option<Arc<dyn ObjectStorageService>> {
fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>> {
None
}

4
frontend/rust-lib/flowy-server/src/server.rs
View File

@ -2,7 +2,6 @@ use client_api::ws::ConnectState;
use client_api::ws::WSConnectStateReceiver;
use client_api::ws::WebSocketChannel;
use flowy_search_pub::cloud::SearchCloudService;
use flowy_storage::ObjectStorageService;
use std::sync::Arc;
use anyhow::Error;
@ -17,6 +16,7 @@ use crate::default_impl::DefaultChatCloudServiceImpl;
use flowy_database_pub::cloud::DatabaseCloudService;
use flowy_document_pub::cloud::DocumentCloudService;
use flowy_folder_pub::cloud::FolderCloudService;
use flowy_storage_pub::cloud::StorageCloudService;
use flowy_user_pub::cloud::UserCloudService;
use flowy_user_pub::entities::UserTokenState;
@ -144,7 +144,7 @@ pub trait AppFlowyServer: Send + Sync + 'static {
Ok(None)
}
fn file_storage(&self) -> Option<Arc<dyn ObjectStorageService>>;
fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>>;
}
pub struct EncryptionImpl {

7
frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs
View File

@ -1,7 +1,5 @@
use std::borrow::Cow;
use anyhow::Error;
use flowy_storage::StorageObject;
use flowy_storage_pub::cloud::StorageObject;
use hyper::header::CONTENT_TYPE;
use reqwest::header::IntoHeaderName;
use reqwest::multipart::{Form, Part};
@ -9,12 +7,14 @@ use reqwest::{
header::{HeaderMap, HeaderValue},
Client, Method, RequestBuilder,
};
use std::borrow::Cow;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use url::Url;
use crate::supabase::file_storage::{DeleteObjects, FileOptions, NewBucket, RequestBody};
#[allow(dead_code)]
pub struct StorageRequestBuilder {
pub url: Url,
headers: HeaderMap,
@ -23,6 +23,7 @@ pub struct StorageRequestBuilder {
body: RequestBody,
}
#[allow(dead_code)]
impl StorageRequestBuilder {
pub fn new(url: Url, headers: HeaderMap, client: Client) -> Self {
Self {

139
frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs
View File

@ -1,141 +1,13 @@
use std::sync::{Arc, Weak};
#![allow(clippy::all)]
#![allow(unknown_lints)]
#![allow(unused_attributes)]
use std::sync::Weak;
use anyhow::{anyhow, Error};
use reqwest::{
header::{HeaderMap, HeaderValue},
Client,
};
use url::Url;
use flowy_encrypt::{decrypt_data, encrypt_data};
use flowy_error::FlowyError;
use flowy_server_pub::supabase_config::SupabaseConfiguration;
use flowy_storage::{FileStoragePlan, ObjectStorageService};
use lib_infra::future::FutureResult;
use crate::supabase::file_storage::builder::StorageRequestBuilder;
use crate::AppFlowyEncryption;
pub struct SupabaseFileStorage {
url: Url,
headers: HeaderMap,
client: Client,
#[allow(dead_code)]
encryption: ObjectEncryption,
#[allow(dead_code)]
storage_plan: Arc<dyn FileStoragePlan>,
}
impl ObjectStorageService for SupabaseFileStorage {
fn get_object_url(
&self,
_object_id: flowy_storage::ObjectIdentity,
) -> FutureResult<String, FlowyError> {
todo!()
}
fn put_object(
&self,
_url: String,
_object_value: flowy_storage::ObjectValue,
) -> FutureResult<(), FlowyError> {
todo!()
}
fn delete_object(&self, _url: String) -> FutureResult<(), FlowyError> {
todo!()
}
fn get_object(&self, _url: String) -> FutureResult<flowy_storage::ObjectValue, FlowyError> {
todo!()
}
// fn create_object(&self, object: StorageObject) -> FutureResult<String, FlowyError> {
// let mut storage = self.storage();
// let storage_plan = Arc::downgrade(&self.storage_plan);
// FutureResult::new(async move {
// let plan = storage_plan
// .upgrade()
// .ok_or(anyhow!("Storage plan is not available"))?;
// plan.check_upload_object(&object).await?;
// storage = storage.upload_object("data", object);
// let url = storage.url.to_string();
// storage.build().await?.send().await?.success().await?;
// Ok(url)
// })
// }
// fn delete_object_by_url(&self, object_url: String) -> FutureResult<(), FlowyError> {
// let storage = self.storage();
// FutureResult::new(async move {
// let url = Url::parse(&object_url)?;
// let location = get_object_location_from(&url)?;
// storage
// .delete_object(location.bucket_id, location.file_name)
// .build()
// .await?
// .send()
// .await?
// .success()
// .await?;
// Ok(())
// })
// }
// fn get_object_by_url(&self, object_url: String) -> FutureResult<Bytes, FlowyError> {
// let storage = self.storage();
// FutureResult::new(async move {
// let url = Url::parse(&object_url)?;
// let location = get_object_location_from(&url)?;
// let bytes = storage
// .get_object(location.bucket_id, location.file_name)
// .build()
// .await?
// .send()
// .await?
// .get_bytes()
// .await?;
// Ok(bytes)
// })
// }
}
impl SupabaseFileStorage {
pub fn new(
config: &SupabaseConfiguration,
encryption: Weak<dyn AppFlowyEncryption>,
storage_plan: Arc<dyn FileStoragePlan>,
) -> Result<Self, Error> {
let mut headers = HeaderMap::new();
let url = format!("{}/storage/v1", config.url);
let auth = format!("Bearer {}", config.anon_key);
headers.insert(
"Authorization",
HeaderValue::from_str(&auth).expect("Authorization is invalid"),
);
headers.insert(
"apikey",
HeaderValue::from_str(&config.anon_key).expect("apikey value is invalid"),
);
let encryption = ObjectEncryption::new(encryption);
Ok(Self {
url: Url::parse(&url)?,
headers,
client: Client::new(),
encryption,
storage_plan,
})
}
pub fn storage(&self) -> StorageRequestBuilder {
StorageRequestBuilder::new(self.url.clone(), self.headers.clone(), self.client.clone())
}
}
use flowy_encrypt::{decrypt_data, encrypt_data};
#[allow(dead_code)]
struct ObjectEncryption {
@ -143,6 +15,7 @@ struct ObjectEncryption {
}
impl ObjectEncryption {
#[allow(dead_code)]
fn new(encryption: Weak<dyn AppFlowyEncryption>) -> Self {
Self { encryption }
}

3
frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs
View File

@ -1,8 +1,7 @@
use bytes::Bytes;
use flowy_storage_pub::cloud::ObjectValueSupabase;
use serde::{Deserialize, Serialize};
use flowy_storage::ObjectValueSupabase;
use crate::supabase;
#[derive(Debug, Clone, Serialize, Deserialize)]

2
frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs
View File

@ -3,7 +3,7 @@ use std::sync::Weak;
use parking_lot::RwLock;
use flowy_error::FlowyError;
use flowy_storage::{FileStoragePlan, StorageObject};
use flowy_storage_pub::cloud::{FileStoragePlan, StorageObject};
use lib_infra::future::FutureResult;
use crate::supabase::api::RESTfulPostgresServer;

38
frontend/rust-lib/flowy-server/src/supabase/server.rs
View File

@ -1,5 +1,4 @@
use flowy_search_pub::cloud::SearchCloudService;
use flowy_storage::ObjectStorageService;
use std::collections::HashMap;
use std::sync::{Arc, Weak};
@ -11,6 +10,7 @@ use flowy_database_pub::cloud::DatabaseCloudService;
use flowy_document_pub::cloud::DocumentCloudService;
use flowy_folder_pub::cloud::FolderCloudService;
use flowy_server_pub::supabase_config::SupabaseConfiguration;
use flowy_storage_pub::cloud::StorageCloudService;
use flowy_user_pub::cloud::UserCloudService;
use crate::supabase::api::{
@ -18,8 +18,7 @@ use crate::supabase::api::{
SupabaseCollabStorageImpl, SupabaseDatabaseServiceImpl, SupabaseDocumentServiceImpl,
SupabaseFolderServiceImpl, SupabaseServerServiceImpl, SupabaseUserServiceImpl,
};
use crate::supabase::file_storage::core::SupabaseFileStorage;
use crate::supabase::file_storage::FileStoragePlanImpl;
use crate::{AppFlowyEncryption, AppFlowyServer};
/// https://www.pgbouncer.org/features.html
@ -63,10 +62,10 @@ pub struct SupabaseServer {
#[allow(dead_code)]
config: SupabaseConfiguration,
device_id: String,
#[allow(dead_code)]
uid: Arc<RwLock<Option<i64>>>,
collab_update_sender: Arc<CollabUpdateSenderByOid>,
restful_postgres: Arc<RwLock<Option<Arc<RESTfulPostgresServer>>>>,
file_storage: Arc<RwLock<Option<Arc<SupabaseFileStorage>>>>,
encryption: Weak<dyn AppFlowyEncryption>,
}
@ -87,23 +86,11 @@ impl SupabaseServer {
} else {
None
};
let file_storage = if enable_sync {
let plan = FileStoragePlanImpl::new(
Arc::downgrade(&uid),
restful_postgres.as_ref().map(Arc::downgrade),
);
Some(Arc::new(
SupabaseFileStorage::new(&config, encryption.clone(), Arc::new(plan)).unwrap(),
))
} else {
None
};
Self {
config,
device_id,
collab_update_sender,
restful_postgres: Arc::new(RwLock::new(restful_postgres)),
file_storage: Arc::new(RwLock::new(file_storage)),
encryption,
uid,
}
@ -119,19 +106,8 @@ impl AppFlowyServer for SupabaseServer {
let postgres = RESTfulPostgresServer::new(self.config.clone(), self.encryption.clone());
*self.restful_postgres.write() = Some(Arc::new(postgres));
}
if self.file_storage.read().is_none() {
let plan = FileStoragePlanImpl::new(
Arc::downgrade(&self.uid),
self.restful_postgres.read().as_ref().map(Arc::downgrade),
);
let file_storage =
SupabaseFileStorage::new(&self.config, self.encryption.clone(), Arc::new(plan)).unwrap();
*self.file_storage.write() = Some(Arc::new(file_storage));
}
} else {
*self.restful_postgres.write() = None;
*self.file_storage.write() = None;
}
}
@ -188,12 +164,8 @@ impl AppFlowyServer for SupabaseServer {
)))
}
fn file_storage(&self) -> Option<Arc<dyn ObjectStorageService>> {
self
.file_storage
.read()
.clone()
.map(|s| s as Arc<dyn ObjectStorageService>)
fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>> {
None
}
fn search_service(&self) -> Option<Arc<dyn SearchCloudService>> {

2
frontend/rust-lib/flowy-server/tests/main.rs
View File

@ -5,7 +5,7 @@ use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
mod af_cloud_test;
mod supabase_test;
// mod supabase_test;
pub fn setup_log() {
static START: Once = Once::new();

21
frontend/rust-lib/flowy-server/tests/supabase_test/util.rs
View File

@ -1,4 +1,3 @@
use flowy_storage::ObjectStorageService;
use std::collections::HashMap;
use std::sync::Arc;
@ -16,10 +15,8 @@ use flowy_server::supabase::api::{
SupabaseFolderServiceImpl, SupabaseServerServiceImpl, SupabaseUserServiceImpl,
};
use flowy_server::supabase::define::{USER_DEVICE_ID, USER_EMAIL, USER_UUID};
use flowy_server::supabase::file_storage::core::SupabaseFileStorage;
use flowy_server::{AppFlowyEncryption, EncryptionImpl};
use flowy_server_pub::supabase_config::SupabaseConfiguration;
use flowy_storage::{FileStoragePlan, StorageObject};
use flowy_user_pub::cloud::UserCloudService;
use lib_infra::future::FutureResult;
@ -63,7 +60,7 @@ pub fn folder_service() -> Arc<dyn FolderCloudService> {
}
#[allow(dead_code)]
pub fn file_storage_service() -> Arc<dyn ObjectStorageService> {
pub fn file_storage_service() -> Arc<dyn ObjectStorageCloudService> {
let encryption_impl: Arc<dyn AppFlowyEncryption> = Arc::new(EncryptionImpl::new(None));
let config = SupabaseConfiguration::from_env().unwrap();
Arc::new(
@ -163,19 +160,3 @@ pub fn third_party_sign_up_param(uuid: String) -> HashMap<String, String> {
}
pub struct TestFileStoragePlan;
impl FileStoragePlan for TestFileStoragePlan {
fn storage_size(&self) -> FutureResult<u64, FlowyError> {
// 1 GB
FutureResult::new(async { Ok(1024 * 1024 * 1024) })
}
fn maximum_file_size(&self) -> FutureResult<u64, FlowyError> {
// 5 MB
FutureResult::new(async { Ok(5 * 1024 * 1024) })
}
fn check_upload_object(&self, _object: &StorageObject) -> FutureResult<(), FlowyError> {
FutureResult::new(async { Ok(()) })
}
}