Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

build doc from local revision or fetch from remote

Browse Source
This commit is contained in:
appflowy
2021-10-02 17:19:54 +08:00
parent 4e3ebf8876
commit af6afafd0f
29 changed files with 764 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/application.rs
View File

@ -114,7 +114,7 @@ fn user_scope() -> Scope {
.route(web::patch().to(view::update_handler))
)
.service(web::resource("/doc")
.route(web::get().to(doc::create_handler))
.route(web::post().to(doc::create_handler))
.route(web::get().to(doc::read_handler))
.route(web::patch().to(doc::update_handler))
)

20
backend/src/service/doc/doc.rs
View File

@ -1,8 +1,8 @@
use crate::service::{
doc::{
actor::{DocWsMsg, DocWsMsgActor},
edit::EditDoc,
edit::DocHandle,
read_doc,
ws_actor::{DocWsActor, DocWsMsg},
},
ws::{WsBizHandler, WsClientData},
};
@ -27,7 +27,7 @@ impl DocBiz {
pub fn new(pg_pool: Data<PgPool>) -> Self {
let manager = Arc::new(DocManager::new());
let (tx, rx) = mpsc::channel(100);
let actor = DocWsMsgActor::new(rx, manager.clone());
let actor = DocWsActor::new(rx, manager.clone());
tokio::task::spawn(actor.run());
Self {
manager,
@ -58,7 +58,7 @@ impl WsBizHandler for DocBiz {
}
pub struct DocManager {
docs_map: DashMap<String, Arc<EditDoc>>,
docs_map: DashMap<String, Arc<DocHandle>>,
}
impl DocManager {
@ -68,7 +68,7 @@ impl DocManager {
}
}
pub async fn get(&self, doc_id: &str, pg_pool: Data<PgPool>) -> Result<Option<Arc<EditDoc>>, ServerError> {
pub async fn get(&self, doc_id: &str, pg_pool: Data<PgPool>) -> Result<Option<Arc<DocHandle>>, ServerError> {
match self.docs_map.get(doc_id) {
None => {
let params = QueryDocParams {
@ -76,10 +76,12 @@ impl DocManager {
..Default::default()
};
let doc = read_doc(pg_pool.get_ref(), params).await?;
let edit_doc = spawn_blocking(|| EditDoc::new(doc)).await.map_err(internal_error)?;
let edit_doc = Arc::new(edit_doc?);
self.docs_map.insert(doc_id.to_string(), edit_doc.clone());
Ok(Some(edit_doc))
let handle = spawn_blocking(|| DocHandle::new(doc, pg_pool))
.await
.map_err(internal_error)?;
let handle = Arc::new(handle?);
self.docs_map.insert(doc_id.to_string(), handle.clone());
Ok(Some(handle))
},
Some(ctx) => Ok(Some(ctx.clone())),
}

194
backend/src/service/doc/edit/context.rs
View File

@ -1,194 +0,0 @@
use crate::service::{doc::edit::actor::EditUser, util::md5, ws::WsMessageAdaptor};
use byteorder::{BigEndian, WriteBytesExt};
use bytes::Bytes;
use dashmap::DashMap;
use flowy_document::{
entities::ws::{WsDataType, WsDocumentData},
protobuf::{Doc, RevType, Revision, RevisionRange, UpdateDocParams},
services::doc::Document,
};
use flowy_net::errors::{internal_error, ServerError};
use flowy_ot::{
core::{Delta, OperationTransformable},
errors::OTError,
};
use flowy_ws::WsMessage;
use parking_lot::RwLock;
use protobuf::Message;
use std::{
convert::TryInto,
sync::{
atomic::{AtomicI64, Ordering::SeqCst},
Arc,
},
time::Duration,
};
pub struct EditDocContext {
doc_id: String,
rev_id: AtomicI64,
document: Arc<RwLock<Document>>,
users: DashMap<String, EditUser>,
}
impl EditDocContext {
pub fn new(doc: Doc) -> Result<Self, ServerError> {
let delta = Delta::from_bytes(&doc.data).map_err(internal_error)?;
let document = Arc::new(RwLock::new(Document::from_delta(delta)));
let users = DashMap::new();
Ok(Self {
doc_id: doc.id.clone(),
rev_id: AtomicI64::new(doc.rev_id),
document,
users,
})
}
pub fn document_json(&self) -> String { self.document.read().to_json() }
pub async fn apply_revision(&self, user: EditUser, revision: Revision) -> Result<(), ServerError> {
// Opti: find out another way to keep the user socket available.
self.users.insert(user.id(), user.clone());
log::debug!(
"cur_base_rev_id: {}, expect_base_rev_id: {} rev_id: {}",
self.rev_id.load(SeqCst),
revision.base_rev_id,
revision.rev_id
);
let cur_rev_id = self.rev_id.load(SeqCst);
if cur_rev_id > revision.rev_id {
// The client document is outdated. Transform the client revision delta and then
// send the prime delta to the client. Client should compose the this prime
// delta.
let (cli_prime, server_prime) = self.transform(&revision.delta_data).map_err(internal_error)?;
let _ = self.update_document_delta(server_prime)?;
log::debug!("{} client delta: {}", self.doc_id, cli_prime.to_json());
let cli_revision = self.mk_revision(revision.rev_id, cli_prime);
let ws_cli_revision = mk_push_rev_ws_message(&self.doc_id, cli_revision);
user.socket.do_send(ws_cli_revision).map_err(internal_error)?;
Ok(())
} else if cur_rev_id < revision.rev_id {
if cur_rev_id != revision.base_rev_id {
// The server document is outdated, try to get the missing revision from the
// client.
user.socket
.do_send(mk_pull_rev_ws_message(&self.doc_id, cur_rev_id, revision.rev_id))
.map_err(internal_error)?;
} else {
let delta = Delta::from_bytes(&revision.delta_data).map_err(internal_error)?;
let _ = self.update_document_delta(delta)?;
user.socket
.do_send(mk_acked_ws_message(&revision))
.map_err(internal_error)?;
self.rev_id.fetch_update(SeqCst, SeqCst, |_e| Some(revision.rev_id));
let _ = self.save_revision(&revision).await?;
}
Ok(())
} else {
log::error!("Client rev_id should not equal to server rev_id");
Ok(())
}
}
fn mk_revision(&self, base_rev_id: i64, delta: Delta) -> Revision {
let delta_data = delta.to_bytes().to_vec();
let md5 = md5(&delta_data);
let revision = Revision {
base_rev_id,
rev_id: self.rev_id.load(SeqCst),
delta_data,
md5,
doc_id: self.doc_id.to_string(),
ty: RevType::Remote,
..Default::default()
};
revision
}
#[tracing::instrument(level = "debug", skip(self, delta_data))]
fn transform(&self, delta_data: &Vec<u8>) -> Result<(Delta, Delta), OTError> {
log::debug!("Document: {}", self.document.read().to_json());
let doc_delta = self.document.read().delta().clone();
let cli_delta = Delta::from_bytes(delta_data)?;
log::debug!("Compose delta: {}", cli_delta);
let (cli_prime, server_prime) = doc_delta.transform(&cli_delta)?;
Ok((cli_prime, server_prime))
}
#[tracing::instrument(level = "debug", skip(self, delta))]
fn update_document_delta(&self, delta: Delta) -> Result<(), ServerError> {
// Opti: push each revision into queue and process it one by one.
match self.document.try_write_for(Duration::from_millis(300)) {
None => {
log::error!("Failed to acquire write lock of document");
},
Some(mut write_guard) => {
let _ = write_guard.compose_delta(&delta).map_err(internal_error)?;
log::debug!("Document: {}", write_guard.to_json());
},
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self, revision))]
async fn save_revision(&self, revision: &Revision) -> Result<(), ServerError> {
// Opti: save with multiple revisions
let mut params = UpdateDocParams::new();
params.set_doc_id(self.doc_id.clone());
params.set_data(self.document.read().to_json());
params.set_rev_id(revision.rev_id);
// let _ = update_doc(self.pg_pool.get_ref(), params).await?;
Ok(())
}
}
fn mk_push_rev_ws_message(doc_id: &str, revision: Revision) -> WsMessageAdaptor {
let bytes = revision.write_to_bytes().unwrap();
let data = WsDocumentData {
id: doc_id.to_string(),
ty: WsDataType::PushRev,
data: bytes,
};
mk_ws_message(data)
}
fn mk_pull_rev_ws_message(doc_id: &str, from_rev_id: i64, to_rev_id: i64) -> WsMessageAdaptor {
let range = RevisionRange {
doc_id: doc_id.to_string(),
from_rev_id,
to_rev_id,
..Default::default()
};
let bytes = range.write_to_bytes().unwrap();
let data = WsDocumentData {
id: doc_id.to_string(),
ty: WsDataType::PullRev,
data: bytes,
};
mk_ws_message(data)
}
fn mk_acked_ws_message(revision: &Revision) -> WsMessageAdaptor {
let mut wtr = vec![];
let _ = wtr.write_i64::<BigEndian>(revision.rev_id);
let data = WsDocumentData {
id: revision.doc_id.clone(),
ty: WsDataType::Acked,
data: wtr,
};
mk_ws_message(data)
}
fn mk_ws_message<T: Into<WsMessage>>(data: T) -> WsMessageAdaptor {
let msg: WsMessage = data.into();
let bytes: Bytes = msg.try_into().unwrap();
WsMessageAdaptor(bytes)
}

25
backend/src/service/doc/edit/actor.rs → backend/src/service/doc/edit/edit_actor.rs
View File

@ -1,11 +1,13 @@
use crate::service::{
doc::edit::EditDocContext,
doc::edit::ServerEditDoc,
ws::{entities::Socket, WsUser},
};
use actix_web::web::Data;
use async_stream::stream;
use flowy_document::protobuf::Revision;
use flowy_net::errors::{internal_error, Result as DocResult};
use flowy_document::protobuf::{Doc, Revision};
use flowy_net::errors::{internal_error, Result as DocResult, ServerError};
use futures::stream::StreamExt;
use sqlx::PgPool;
use std::sync::Arc;
use tokio::{
sync::{mpsc, oneshot},
@ -37,15 +39,18 @@ pub enum EditMsg {
pub struct EditDocActor {
receiver: Option<mpsc::Receiver<EditMsg>>,
edit_context: Arc<EditDocContext>,
edit_doc: Arc<ServerEditDoc>,
pg_pool: Data<PgPool>,
}
impl EditDocActor {
pub fn new(receiver: mpsc::Receiver<EditMsg>, edit_context: Arc<EditDocContext>) -> Self {
Self {
pub fn new(receiver: mpsc::Receiver<EditMsg>, doc: Doc, pg_pool: Data<PgPool>) -> Result<Self, ServerError> {
let edit_doc = Arc::new(ServerEditDoc::new(doc)?);
Ok(Self {
receiver: Some(receiver),
edit_context,
}
edit_doc,
pg_pool,
})
}
pub async fn run(mut self) {
@ -78,10 +83,10 @@ impl EditDocActor {
user: user.clone(),
socket: socket.clone(),
};
let _ = ret.send(self.edit_context.apply_revision(user, revision).await);
let _ = ret.send(self.edit_doc.apply_revision(user, revision, self.pg_pool.clone()).await);
},
EditMsg::DocumentJson { ret } => {
let edit_context = self.edit_context.clone();
let edit_context = self.edit_doc.clone();
let json = spawn_blocking(move || edit_context.document_json())
.await
.map_err(internal_error);

228
backend/src/service/doc/edit/edit_doc.rs
View File

@ -1,56 +1,206 @@
use crate::service::{
doc::edit::{
actor::{EditDocActor, EditMsg},
EditDocContext,
},
ws::{entities::Socket, WsUser},
doc::{edit::edit_actor::EditUser, update_doc},
util::md5,
ws::WsMessageAdaptor,
};
use flowy_document::protobuf::{Doc, Revision};
use flowy_net::errors::{internal_error, Result as DocResult, ServerError};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
pub struct EditDoc {
sender: mpsc::Sender<EditMsg>,
use actix_web::web::Data;
use byteorder::{BigEndian, WriteBytesExt};
use bytes::Bytes;
use dashmap::DashMap;
use flowy_document::{
entities::ws::{WsDataType, WsDocumentData},
protobuf::{Doc, RevType, Revision, RevisionRange, UpdateDocParams},
services::doc::Document,
};
use flowy_net::errors::{internal_error, ServerError};
use flowy_ot::{
core::{Delta, OperationTransformable},
errors::OTError,
};
use flowy_ws::WsMessage;
use parking_lot::RwLock;
use protobuf::Message;
use sqlx::PgPool;
use std::{
convert::TryInto,
sync::{
atomic::{AtomicI64, Ordering::SeqCst},
Arc,
},
time::Duration,
};
pub struct ServerEditDoc {
doc_id: String,
rev_id: AtomicI64,
document: Arc<RwLock<Document>>,
users: DashMap<String, EditUser>,
}
impl EditDoc {
impl ServerEditDoc {
pub fn new(doc: Doc) -> Result<Self, ServerError> {
let (sender, receiver) = mpsc::channel(100);
let edit_context = Arc::new(EditDocContext::new(doc)?);
let actor = EditDocActor::new(receiver, edit_context);
tokio::task::spawn(actor.run());
Ok(Self { sender })
let delta = Delta::from_bytes(&doc.data).map_err(internal_error)?;
let document = Arc::new(RwLock::new(Document::from_delta(delta)));
let users = DashMap::new();
Ok(Self {
doc_id: doc.id.clone(),
rev_id: AtomicI64::new(doc.rev_id),
document,
users,
})
}
#[tracing::instrument(level = "debug", skip(self, user, socket, revision))]
pub fn document_json(&self) -> String { self.document.read().to_json() }
pub async fn apply_revision(
&self,
user: Arc<WsUser>,
socket: Socket,
user: EditUser,
revision: Revision,
pg_pool: Data<PgPool>,
) -> Result<(), ServerError> {
let (ret, rx) = oneshot::channel();
let msg = EditMsg::Revision {
user,
socket,
revision,
ret,
// Opti: find out another way to keep the user socket available.
self.users.insert(user.id(), user.clone());
log::debug!(
"cur_base_rev_id: {}, expect_base_rev_id: {} rev_id: {}",
self.rev_id.load(SeqCst),
revision.base_rev_id,
revision.rev_id
);
let cur_rev_id = self.rev_id.load(SeqCst);
if cur_rev_id > revision.rev_id {
// The client document is outdated. Transform the client revision delta and then
// send the prime delta to the client. Client should compose the this prime
// delta.
let (cli_prime, server_prime) = self.transform(&revision.delta_data).map_err(internal_error)?;
let _ = self.update_document_delta(server_prime)?;
log::debug!("{} client delta: {}", self.doc_id, cli_prime.to_json());
let cli_revision = self.mk_revision(revision.rev_id, cli_prime);
let ws_cli_revision = mk_push_rev_ws_message(&self.doc_id, cli_revision);
user.socket.do_send(ws_cli_revision).map_err(internal_error)?;
Ok(())
} else if cur_rev_id < revision.rev_id {
if cur_rev_id != revision.base_rev_id {
// The server document is outdated, try to get the missing revision from the
// client.
user.socket
.do_send(mk_pull_rev_ws_message(&self.doc_id, cur_rev_id, revision.rev_id))
.map_err(internal_error)?;
} else {
let delta = Delta::from_bytes(&revision.delta_data).map_err(internal_error)?;
let _ = self.update_document_delta(delta)?;
user.socket
.do_send(mk_acked_ws_message(&revision))
.map_err(internal_error)?;
self.rev_id.fetch_update(SeqCst, SeqCst, |_e| Some(revision.rev_id));
let _ = self.save_revision(&revision, pg_pool).await?;
}
Ok(())
} else {
log::error!("Client rev_id should not equal to server rev_id");
Ok(())
}
}
fn mk_revision(&self, base_rev_id: i64, delta: Delta) -> Revision {
let delta_data = delta.to_bytes().to_vec();
let md5 = md5(&delta_data);
let revision = Revision {
base_rev_id,
rev_id: self.rev_id.load(SeqCst),
delta_data,
md5,
doc_id: self.doc_id.to_string(),
ty: RevType::Remote,
..Default::default()
};
let _ = self.send(msg, rx).await?;
revision
}
#[tracing::instrument(level = "debug", skip(self, delta_data))]
fn transform(&self, delta_data: &Vec<u8>) -> Result<(Delta, Delta), OTError> {
log::debug!("Document: {}", self.document.read().to_json());
let doc_delta = self.document.read().delta().clone();
let cli_delta = Delta::from_bytes(delta_data)?;
log::debug!("Compose delta: {}", cli_delta);
let (cli_prime, server_prime) = doc_delta.transform(&cli_delta)?;
Ok((cli_prime, server_prime))
}
#[tracing::instrument(level = "debug", skip(self), err)]
fn update_document_delta(&self, delta: Delta) -> Result<(), ServerError> {
// Opti: push each revision into queue and process it one by one.
match self.document.try_write_for(Duration::from_millis(300)) {
None => {
log::error!("Failed to acquire write lock of document");
},
Some(mut write_guard) => {
let _ = write_guard.compose_delta(&delta).map_err(internal_error)?;
log::debug!("Document: {}", write_guard.to_json());
},
}
Ok(())
}
pub async fn document_json(&self) -> DocResult<String> {
let (ret, rx) = oneshot::channel();
let msg = EditMsg::DocumentJson { ret };
self.send(msg, rx).await?
}
async fn send<T>(&self, msg: EditMsg, rx: oneshot::Receiver<T>) -> DocResult<T> {
let _ = self.sender.send(msg).await.map_err(internal_error)?;
let result = rx.await?;
Ok(result)
#[tracing::instrument(level = "debug", skip(self, pg_pool), err)]
async fn save_revision(&self, revision: &Revision, pg_pool: Data<PgPool>) -> Result<(), ServerError> {
// Opti: save with multiple revisions
let mut params = UpdateDocParams::new();
params.set_doc_id(self.doc_id.clone());
params.set_data(self.document.read().to_json());
params.set_rev_id(revision.rev_id);
let _ = update_doc(pg_pool.get_ref(), params).await?;
Ok(())
}
}
fn mk_push_rev_ws_message(doc_id: &str, revision: Revision) -> WsMessageAdaptor {
let bytes = revision.write_to_bytes().unwrap();
let data = WsDocumentData {
id: doc_id.to_string(),
ty: WsDataType::PushRev,
data: bytes,
};
mk_ws_message(data)
}
fn mk_pull_rev_ws_message(doc_id: &str, from_rev_id: i64, to_rev_id: i64) -> WsMessageAdaptor {
let range = RevisionRange {
doc_id: doc_id.to_string(),
from_rev_id,
to_rev_id,
..Default::default()
};
let bytes = range.write_to_bytes().unwrap();
let data = WsDocumentData {
id: doc_id.to_string(),
ty: WsDataType::PullRev,
data: bytes,
};
mk_ws_message(data)
}
fn mk_acked_ws_message(revision: &Revision) -> WsMessageAdaptor {
let mut wtr = vec![];
let _ = wtr.write_i64::<BigEndian>(revision.rev_id);
let data = WsDocumentData {
id: revision.doc_id.clone(),
ty: WsDataType::Acked,
data: wtr,
};
mk_ws_message(data)
}
fn mk_ws_message<T: Into<WsMessage>>(data: T) -> WsMessageAdaptor {
let msg: WsMessage = data.into();
let bytes: Bytes = msg.try_into().unwrap();
WsMessageAdaptor(bytes)
}

6
backend/src/service/doc/edit/mod.rs
View File

@ -1,6 +1,6 @@
mod actor;
mod context;
mod edit_actor;
mod edit_doc;
mod open_handle;
pub use context::*;
pub use edit_doc::*;
pub use open_handle::*;

57
backend/src/service/doc/edit/open_handle.rs Normal file
View File

@ -0,0 +1,57 @@
use crate::service::{
doc::edit::{
edit_actor::{EditDocActor, EditMsg},
ServerEditDoc,
},
ws::{entities::Socket, WsUser},
};
use actix_web::web::Data;
use flowy_document::protobuf::{Doc, Revision};
use flowy_net::errors::{internal_error, Result as DocResult, ServerError};
use sqlx::PgPool;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
pub struct DocHandle {
sender: mpsc::Sender<EditMsg>,
}
impl DocHandle {
pub fn new(doc: Doc, pg_pool: Data<PgPool>) -> Result<Self, ServerError> {
let (sender, receiver) = mpsc::channel(100);
let actor = EditDocActor::new(receiver, doc, pg_pool)?;
tokio::task::spawn(actor.run());
Ok(Self { sender })
}
#[tracing::instrument(level = "debug", skip(self, user, socket, revision))]
pub async fn apply_revision(
&self,
user: Arc<WsUser>,
socket: Socket,
revision: Revision,
) -> Result<(), ServerError> {
let (ret, rx) = oneshot::channel();
let msg = EditMsg::Revision {
user,
socket,
revision,
ret,
};
let _ = self.send(msg, rx).await?;
Ok(())
}
pub async fn document_json(&self) -> DocResult<String> {
let (ret, rx) = oneshot::channel();
let msg = EditMsg::DocumentJson { ret };
self.send(msg, rx).await?
}
async fn send<T>(&self, msg: EditMsg, rx: oneshot::Receiver<T>) -> DocResult<T> {
let _ = self.sender.send(msg).await.map_err(internal_error)?;
let result = rx.await?;
Ok(result)
}
}

2
backend/src/service/doc/mod.rs
View File

@ -1,8 +1,8 @@
pub(crate) use crud::*;
pub use router::*;
mod actor;
pub mod crud;
pub mod doc;
mod edit;
pub mod router;
mod ws_actor;

1
backend/src/service/doc/router.rs
View File

@ -29,6 +29,7 @@ pub async fn create_handler(payload: Payload, pool: Data<PgPool>) -> Result<Http
Ok(FlowyResponse::success().into())
}
#[tracing::instrument(level = "debug", skip(payload, pool), err)]
pub async fn read_handler(payload: Payload, pool: Data<PgPool>) -> Result<HttpResponse, ServerError> {
let params: QueryDocParams = parse_from_payload(payload).await?;
let doc = read_doc(pool.get_ref(), params).await?;

4
backend/src/service/doc/actor.rs → backend/src/service/doc/ws_actor.rs
View File

@ -21,12 +21,12 @@ pub enum DocWsMsg {
},
}
pub struct DocWsMsgActor {
pub struct DocWsActor {
receiver: Option<mpsc::Receiver<DocWsMsg>>,
doc_manager: Arc<DocManager>,
}
impl DocWsMsgActor {
impl DocWsActor {
pub fn new(receiver: mpsc::Receiver<DocWsMsg>, manager: Arc<DocManager>) -> Self {
Self {
receiver: Some(receiver),

2
backend/tests/document/helper.rs
View File

@ -6,7 +6,7 @@ use sqlx::PgPool;
use tokio::time::{sleep, Duration};
use backend::service::doc::doc::DocManager;
use flowy_document::{entities::doc::QueryDocParams, services::doc::edit::EditDocContext as ClientEditDocContext};
use flowy_document::{entities::doc::QueryDocParams, services::doc::edit::ClientEditDoc as ClientEditDocContext};
use flowy_net::config::ServerConfig;
use flowy_test::{workspace::ViewTest, FlowyTest};
use flowy_user::services::user::UserSession;