refactor: refactor some crates with http_server

frontend/rust-lib/.gitignore

@@ -4,7 +4,6 @@
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

frontend/rust-lib/flowy-document/Cargo.toml

@@ -17,7 +17,7 @@ derive_more = {version = "0.99", features = ["display"]}
 lib-dispatch = { path = "../lib-dispatch" }
 flowy-database = { path = "../flowy-database" }
 flowy-sync = { path = "../flowy-sync" }
-flowy-error = { path = "../flowy-error", features = ["collaboration", "ot", "backend", "serde", "db"] }
+flowy-error = { path = "../flowy-error", features = ["collaboration", "ot", "http_server", "serde", "db"] }
 dart-notify = { path = "../dart-notify" }
 
 diesel = {version = "1.4.8", features = ["sqlite"]}

frontend/rust-lib/flowy-error/Cargo.toml

@@ -16,7 +16,7 @@ bytes = "1.0"
 flowy-collaboration = { path = "../../../shared-lib/flowy-collaboration", optional = true}
 lib-ot = { path = "../../../shared-lib/lib-ot", optional = true}
 serde_json = {version = "1.0", optional = true}
-backend-service = { path = "../../../shared-lib/backend-service", optional = true}
+http-flowy = { git = "https://github.com/AppFlowy-IO/AppFlowy-Server", optional = true}
 flowy-database = { path = "../flowy-database", optional = true}
 r2d2 = { version = "0.8", optional = true}
 lib-sqlite = { path = "../lib-sqlite", optional = true }
@@ -25,5 +25,5 @@ lib-sqlite = { path = "../lib-sqlite", optional = true }
 collaboration = ["flowy-collaboration"]
 ot = ["lib-ot"]
 serde = ["serde_json"]
-backend = ["backend-service"]
+http_server = ["http-flowy"]
 db = ["flowy-database", "lib-sqlite", "r2d2"]

frontend/rust-lib/flowy-error/src/ext/http_server.rs

@@ -1,6 +1,6 @@
 use crate::FlowyError;
-use backend_service::errors::{ErrorCode as ServerErrorCode, ServerError};
 use error_code::ErrorCode;
+use http_flowy::errors::{ErrorCode as ServerErrorCode, ServerError};
 
 impl std::convert::From<ServerError> for FlowyError {
     fn from(error: ServerError) -> Self {

frontend/rust-lib/flowy-error/src/ext/mod.rs

@@ -16,10 +16,10 @@ mod serde;
 pub use serde::*;
 
 //
-#[cfg(feature = "backend")]
-mod backend;
-#[cfg(feature = "backend")]
-pub use backend::*;
+#[cfg(feature = "http_server")]
+mod http_server;
+#[cfg(feature = "http_server")]
+pub use http_server::*;
 
 #[cfg(feature = "db")]
 mod database;

frontend/rust-lib/flowy-folder/Cargo.toml

@@ -14,7 +14,7 @@ lib-infra = { path = "../../../shared-lib/lib-infra" }
 
 flowy-document = { path = "../flowy-document" }
 flowy-database = { path = "../flowy-database" }
-flowy-error = { path = "../flowy-error", features = ["db", "backend"]}
+flowy-error = { path = "../flowy-error", features = ["db", "http_server"]}
 dart-notify = { path = "../dart-notify" }
 lib-dispatch = { path = "../lib-dispatch" }
 lib-sqlite = { path = "../lib-sqlite" }

frontend/rust-lib/flowy-net/Cargo.toml

@@ -7,10 +7,9 @@ edition = "2018"
 
 [dependencies]
 lib-dispatch = { path = "../lib-dispatch" }
-flowy-error = { path = "../flowy-error", features = ["collaboration"] }
+flowy-error = { path = "../flowy-error", features = ["collaboration", "http_server"] }
 flowy-derive = { path = "../../../shared-lib/flowy-derive" }
 flowy-collaboration = { path = "../../../shared-lib/flowy-collaboration"}
-backend-service = { path = "../../../shared-lib/backend-service" }
 flowy-folder-data-model = { path = "../../../shared-lib/flowy-folder-data-model" }
 flowy-user-data-model = { path = "../../../shared-lib/flowy-user-data-model"}
 flowy-folder = { path = "../flowy-folder" }
@@ -30,5 +29,14 @@ tracing = { version = "0.1", features = ["log"] }
 dashmap = {version = "4.0"}
 async-stream = "0.3.2"
 futures-util = "0.3.15"
+http-flowy = { git = "https://github.com/AppFlowy-IO/AppFlowy-Server", features = ["with_reqwest"] }
+serde-aux = "1.0.1"
+reqwest = "0.11"
+hyper = "0.14"
+config = { version = "0.10.1", default-features = false, features = ["yaml"] }
+log = "0.4.14"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
 [features]
 http_server = []

frontend/rust-lib/flowy-net/configuration/base.yaml

@@ -0,0 +1,5 @@
+port: 8000
+host: 0.0.0.0
+http_scheme: http
+ws_scheme: ws
+

frontend/rust-lib/flowy-net/configuration/local.yaml

@@ -0,0 +1,3 @@
+host: 127.0.0.1
+http_scheme: http
+ws_scheme: ws

frontend/rust-lib/flowy-net/configuration/production.yaml

@@ -0,0 +1,2 @@
+host: 0.0.0.0
+

frontend/rust-lib/flowy-net/src/configuration.rs

@@ -0,0 +1,113 @@
+use config::FileFormat;
+use serde_aux::field_attributes::deserialize_number_from_string;
+use std::convert::{TryFrom, TryInto};
+pub const HEADER_TOKEN: &str = "token";
+
+#[derive(serde::Deserialize, Clone, Debug)]
+pub struct ClientServerConfiguration {
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub port: u16,
+    pub host: String,
+    pub http_scheme: String,
+    pub ws_scheme: String,
+}
+
+pub fn get_client_server_configuration() -> Result<ClientServerConfiguration, config::ConfigError> {
+    let mut settings = config::Config::default();
+    let base = include_str!("../configuration/base.yaml");
+    settings.merge(config::File::from_str(base, FileFormat::Yaml).required(true))?;
+
+    let environment: Environment = std::env::var("APP_ENVIRONMENT")
+        .unwrap_or_else(|_| "local".into())
+        .try_into()
+        .expect("Failed to parse APP_ENVIRONMENT.");
+
+    let custom = match environment {
+        Environment::Local => include_str!("../configuration/local.yaml"),
+        Environment::Production => include_str!("../configuration/production.yaml"),
+    };
+
+    settings.merge(config::File::from_str(custom, FileFormat::Yaml).required(true))?;
+    settings.try_into()
+}
+
+impl ClientServerConfiguration {
+    pub fn reset_host_with_port(&mut self, host: &str, port: u16) {
+        self.host = host.to_owned();
+        self.port = port;
+    }
+
+    pub fn base_url(&self) -> String {
+        format!("{}://{}:{}", self.http_scheme, self.host, self.port)
+    }
+
+    pub fn sign_up_url(&self) -> String {
+        format!("{}/api/register", self.base_url())
+    }
+
+    pub fn sign_in_url(&self) -> String {
+        format!("{}/api/auth", self.base_url())
+    }
+
+    pub fn sign_out_url(&self) -> String {
+        format!("{}/api/auth", self.base_url())
+    }
+
+    pub fn user_profile_url(&self) -> String {
+        format!("{}/api/user", self.base_url())
+    }
+
+    pub fn workspace_url(&self) -> String {
+        format!("{}/api/workspace", self.base_url())
+    }
+
+    pub fn app_url(&self) -> String {
+        format!("{}/api/app", self.base_url())
+    }
+
+    pub fn view_url(&self) -> String {
+        format!("{}/api/view", self.base_url())
+    }
+
+    pub fn doc_url(&self) -> String {
+        format!("{}/api/doc", self.base_url())
+    }
+
+    pub fn trash_url(&self) -> String {
+        format!("{}/api/trash", self.base_url())
+    }
+
+    pub fn ws_addr(&self) -> String {
+        format!("{}://{}:{}/ws", self.ws_scheme, self.host, self.port)
+    }
+}
+
+pub enum Environment {
+    Local,
+    Production,
+}
+
+impl Environment {
+    #[allow(dead_code)]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Environment::Local => "local",
+            Environment::Production => "production",
+        }
+    }
+}
+
+impl TryFrom<String> for Environment {
+    type Error = String;
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        match s.to_lowercase().as_str() {
+            "local" => Ok(Self::Local),
+            "production" => Ok(Self::Production),
+            other => Err(format!(
+                "{} is not a supported environment. Use either `local` or `production`.",
+                other
+            )),
+        }
+    }
+}

frontend/rust-lib/flowy-net/src/http_server/document.rs

@@ -1,11 +1,11 @@
-use backend_service::{
+use crate::{
     configuration::*,
     request::{HttpRequestBuilder, ResponseMiddleware},
-    response::FlowyResponse,
 };
 use flowy_collaboration::entities::document_info::{CreateDocParams, DocumentId, DocumentInfo, ResetDocumentParams};
 use flowy_document::DocumentCloudService;
 use flowy_error::FlowyError;
+use http_flowy::response::FlowyResponse;
 use lazy_static::lazy_static;
 use lib_infra::future::FutureResult;
 use std::sync::Arc;

frontend/rust-lib/flowy-net/src/http_server/folder.rs

@@ -1,8 +1,6 @@
-use backend_service::{
+use crate::{
     configuration::{ClientServerConfiguration, HEADER_TOKEN},
-    errors::ServerError,
     request::{HttpRequestBuilder, ResponseMiddleware},
-    response::FlowyResponse,
 };
 use flowy_error::FlowyError;
 use flowy_folder_data_model::entities::{
@@ -13,6 +11,8 @@ use flowy_folder_data_model::entities::{
 };
 
 use flowy_folder::event_map::FolderCouldServiceV1;
+use http_flowy::errors::ServerError;
+use http_flowy::response::FlowyResponse;
 use lazy_static::lazy_static;
 use lib_infra::future::FutureResult;
 use std::sync::Arc;
@@ -338,17 +338,17 @@ pub async fn read_trash_request(token: &str, url: &str) -> Result<RepeatedTrash,
 }
 
 lazy_static! {
-    static ref MIDDLEWARE: Arc<CoreResponseMiddleware> = Arc::new(CoreResponseMiddleware::new());
+    static ref MIDDLEWARE: Arc<FolderResponseMiddleware> = Arc::new(FolderResponseMiddleware::new());
 }
 
-pub struct CoreResponseMiddleware {
+pub struct FolderResponseMiddleware {
     invalid_token_sender: broadcast::Sender<String>,
 }
 
-impl CoreResponseMiddleware {
+impl FolderResponseMiddleware {
     fn new() -> Self {
         let (sender, _) = broadcast::channel(10);
-        CoreResponseMiddleware {
+        FolderResponseMiddleware {
             invalid_token_sender: sender,
         }
     }
@@ -359,7 +359,7 @@ impl CoreResponseMiddleware {
     }
 }
 
-impl ResponseMiddleware for CoreResponseMiddleware {
+impl ResponseMiddleware for FolderResponseMiddleware {
     fn receive_response(&self, token: &Option<String>, response: &FlowyResponse) {
         if let Some(error) = &response.error {
             if error.is_unauthorized() {

frontend/rust-lib/flowy-net/src/http_server/mod.rs

@@ -1,3 +1,3 @@
-pub mod core;
 pub mod document;
+pub mod folder;
 pub mod user;

frontend/rust-lib/flowy-net/src/http_server/user.rs

@@ -1,9 +1,10 @@
-use backend_service::{configuration::*, errors::ServerError, request::HttpRequestBuilder};
+use crate::{configuration::*, request::HttpRequestBuilder};
 use flowy_error::FlowyError;
 use flowy_user::event_map::UserCloudService;
 use flowy_user_data_model::entities::{
     SignInParams, SignInResponse, SignUpParams, SignUpResponse, UpdateUserParams, UserProfile,
 };
+use http_flowy::errors::ServerError;
 use lib_infra::future::FutureResult;
 
 pub struct UserHttpCloudService {

frontend/rust-lib/flowy-net/src/lib.rs

@@ -1,3 +1,4 @@
+mod configuration;
 pub mod entities;
 mod event;
 mod handlers;
@@ -5,6 +6,7 @@ pub mod http_server;
 pub mod local_server;
 pub mod module;
 pub mod protobuf;
+mod request;
 pub mod ws;
 
-pub use backend_service::configuration::{get_client_server_configuration, ClientServerConfiguration};
+pub use crate::configuration::{get_client_server_configuration, ClientServerConfiguration};

frontend/rust-lib/flowy-net/src/local_server/mod.rs

@@ -1,4 +1,4 @@
-use backend_service::configuration::ClientServerConfiguration;
+use crate::configuration::ClientServerConfiguration;
 use tokio::sync::{broadcast, mpsc};
 
 mod persistence;

frontend/rust-lib/flowy-net/src/request.rs

@@ -0,0 +1,216 @@
+use crate::configuration::HEADER_TOKEN;
+use bytes::Bytes;
+use http_flowy::errors::ServerError;
+use http_flowy::response::FlowyResponse;
+use hyper::http;
+use protobuf::ProtobufError;
+use reqwest::{header::HeaderMap, Client, Method, Response};
+use std::{
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+    time::Duration,
+};
+use tokio::sync::oneshot;
+
+pub trait ResponseMiddleware {
+    fn receive_response(&self, token: &Option<String>, response: &FlowyResponse);
+}
+
+pub struct HttpRequestBuilder {
+    url: String,
+    body: Option<Bytes>,
+    response: Option<Bytes>,
+    headers: HeaderMap,
+    method: Method,
+    middleware: Vec<Arc<dyn ResponseMiddleware + Send + Sync>>,
+}
+
+impl std::default::Default for HttpRequestBuilder {
+    fn default() -> Self {
+        Self {
+            url: "".to_owned(),
+            body: None,
+            response: None,
+            headers: HeaderMap::new(),
+            method: Method::GET,
+            middleware: Vec::new(),
+        }
+    }
+}
+
+impl HttpRequestBuilder {
+    pub fn new() -> Self {
+        HttpRequestBuilder::default()
+    }
+
+    pub fn middleware<T>(mut self, middleware: Arc<T>) -> Self
+    where
+        T: 'static + ResponseMiddleware + Send + Sync,
+    {
+        self.middleware.push(middleware);
+        self
+    }
+
+    pub fn get(mut self, url: &str) -> Self {
+        self.url = url.to_owned();
+        self.method = Method::GET;
+        self
+    }
+
+    pub fn post(mut self, url: &str) -> Self {
+        self.url = url.to_owned();
+        self.method = Method::POST;
+        self
+    }
+
+    pub fn patch(mut self, url: &str) -> Self {
+        self.url = url.to_owned();
+        self.method = Method::PATCH;
+        self
+    }
+
+    pub fn delete(mut self, url: &str) -> Self {
+        self.url = url.to_owned();
+        self.method = Method::DELETE;
+        self
+    }
+
+    pub fn header(mut self, key: &'static str, value: &str) -> Self {
+        self.headers.insert(key, value.parse().unwrap());
+        self
+    }
+
+    pub fn protobuf<T>(self, body: T) -> Result<Self, ServerError>
+    where
+        T: TryInto<Bytes, Error = ProtobufError>,
+    {
+        let body: Bytes = body.try_into()?;
+        self.bytes(body)
+    }
+
+    pub fn bytes(mut self, body: Bytes) -> Result<Self, ServerError> {
+        self.body = Some(body);
+        Ok(self)
+    }
+
+    pub async fn send(self) -> Result<(), ServerError> {
+        let _ = self.inner_send().await?;
+        Ok(())
+    }
+
+    pub async fn response<T>(self) -> Result<T, ServerError>
+    where
+        T: TryFrom<Bytes, Error = ProtobufError>,
+    {
+        let builder = self.inner_send().await?;
+        match builder.response {
+            None => Err(unexpected_empty_payload(&builder.url)),
+            Some(data) => Ok(T::try_from(data)?),
+        }
+    }
+
+    pub async fn option_response<T>(self) -> Result<Option<T>, ServerError>
+    where
+        T: TryFrom<Bytes, Error = ProtobufError>,
+    {
+        let result = self.inner_send().await;
+        match result {
+            Ok(builder) => match builder.response {
+                None => Err(unexpected_empty_payload(&builder.url)),
+                Some(data) => Ok(Some(T::try_from(data)?)),
+            },
+            Err(error) => match error.is_record_not_found() {
+                true => Ok(None),
+                false => Err(error),
+            },
+        }
+    }
+
+    fn token(&self) -> Option<String> {
+        match self.headers.get(HEADER_TOKEN) {
+            None => None,
+            Some(header) => match header.to_str() {
+                Ok(val) => Some(val.to_owned()),
+                Err(_) => None,
+            },
+        }
+    }
+
+    async fn inner_send(mut self) -> Result<Self, ServerError> {
+        let (tx, rx) = oneshot::channel::<Result<Response, _>>();
+        let url = self.url.clone();
+        let body = self.body.take();
+        let method = self.method.clone();
+        let headers = self.headers.clone();
+
+        // reqwest client is not 'Sync' but channel is.
+        tokio::spawn(async move {
+            let client = default_client();
+            let mut builder = client.request(method.clone(), url).headers(headers);
+            if let Some(body) = body {
+                builder = builder.body(body);
+            }
+
+            let response = builder.send().await;
+            let _ = tx.send(response);
+        });
+
+        let response = rx.await.map_err(|e| {
+            let mag = format!("Receive http response channel error: {}", e);
+            ServerError::internal().context(mag)
+        })??;
+        tracing::trace!("Http Response: {:?}", response);
+        let flowy_response = flowy_response_from(response).await?;
+        let token = self.token();
+        self.middleware.iter().for_each(|middleware| {
+            middleware.receive_response(&token, &flowy_response);
+        });
+        match flowy_response.error {
+            None => {
+                self.response = Some(flowy_response.data);
+                Ok(self)
+            }
+            Some(error) => Err(error),
+        }
+    }
+}
+
+fn unexpected_empty_payload(url: &str) -> ServerError {
+    let msg = format!("Request: {} receives unexpected empty payload", url);
+    ServerError::payload_none().context(msg)
+}
+
+async fn flowy_response_from(original: Response) -> Result<FlowyResponse, ServerError> {
+    let bytes = original.bytes().await?;
+    let response: FlowyResponse = serde_json::from_slice(&bytes)?;
+    Ok(response)
+}
+
+#[allow(dead_code)]
+async fn get_response_data(original: Response) -> Result<Bytes, ServerError> {
+    if original.status() == http::StatusCode::OK {
+        let bytes = original.bytes().await?;
+        let response: FlowyResponse = serde_json::from_slice(&bytes)?;
+        match response.error {
+            None => Ok(response.data),
+            Some(error) => Err(error),
+        }
+    } else {
+        Err(ServerError::http().context(original))
+    }
+}
+
+fn default_client() -> Client {
+    let result = reqwest::Client::builder()
+        .connect_timeout(Duration::from_millis(500))
+        .timeout(Duration::from_secs(5))
+        .build();
+
+    match result {
+        Ok(client) => client,
+        Err(e) => {
+            log::error!("Create reqwest client failed: {}", e);
+            reqwest::Client::new()
+        }
+    }
+}

frontend/rust-lib/flowy-sdk/src/deps_resolve/folder_deps.rs

@@ -9,7 +9,7 @@ use flowy_folder::{
 };
 use flowy_net::ClientServerConfiguration;
 use flowy_net::{
-    http_server::core::FolderHttpCloudService, local_server::LocalServer, ws::connection::FlowyWebSocketConnect,
+    http_server::folder::FolderHttpCloudService, local_server::LocalServer, ws::connection::FlowyWebSocketConnect,
 };
 use flowy_sync::{RevisionWebSocket, WSStateReceiver};
 use flowy_user::services::UserSession;

frontend/rust-lib/flowy-sync/Cargo.toml

@@ -11,7 +11,7 @@ lib-ot = { path = "../../../shared-lib/lib-ot" }
 lib-ws = { path = "../../../shared-lib/lib-ws" }
 lib-infra = { path = "../../../shared-lib/lib-infra" }
 flowy-database = { path = "../flowy-database" }
-flowy-error = { path = "../flowy-error", features = ["collaboration", "ot", "backend", "serde", "db"] }
+flowy-error = { path = "../flowy-error", features = ["collaboration", "ot", "http_server", "serde", "db"] }
 diesel = {version = "1.4.8", features = ["sqlite"]}
 diesel_derives = {version = "1.4.1", features = ["sqlite"]}
 protobuf = {version = "2.18.0"}

frontend/rust-lib/flowy-user/Cargo.toml

@@ -14,7 +14,7 @@ derive_more = {version = "0.99", features = ["display"]}
 flowy-database = { path = "../flowy-database" }
 dart-notify = { path = "../dart-notify" }
 lib-dispatch = { path = "../lib-dispatch" }
-flowy-error = { path = "../flowy-error", features = ["db", "backend"] }
+flowy-error = { path = "../flowy-error", features = ["db", "http_server"] }
 lib-sqlite = { path = "../lib-sqlite" }
 
 tracing = { version = "0.1", features = ["log"] }