diff --git a/.gitignore b/.gitignore index 7f594b1012..b74dd0a47e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ package-lock.json yarn.lock node_modules -**/.proto_cache \ No newline at end of file +**/.proto_cache +**/.cache \ No newline at end of file diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 0906fe5ede..2886cd7a1b 100755 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -906,7 +906,9 @@ dependencies = [ "lib-infra", "proc-macro2", "quote", + "serde_json", "syn", + "walkdir", ] [[package]] diff --git a/frontend/rust-lib/dart-ffi/build.rs b/frontend/rust-lib/dart-ffi/build.rs index 4fd025c595..ef2b12f93d 100644 --- a/frontend/rust-lib/dart-ffi/build.rs +++ b/frontend/rust-lib/dart-ffi/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("dart-ffi", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/frontend/rust-lib/dart-notify/build.rs b/frontend/rust-lib/dart-notify/build.rs index aef5af23f6..ef2b12f93d 100644 --- a/frontend/rust-lib/dart-notify/build.rs +++ b/frontend/rust-lib/dart-notify/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("dart-notify", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/frontend/rust-lib/flowy-folder/build.rs b/frontend/rust-lib/flowy-folder/build.rs index 7832bdb707..ef2b12f93d 100644 --- a/frontend/rust-lib/flowy-folder/build.rs +++ b/frontend/rust-lib/flowy-folder/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-folder", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/frontend/rust-lib/flowy-net/build.rs b/frontend/rust-lib/flowy-net/build.rs index 002d156074..ef2b12f93d 100644 --- a/frontend/rust-lib/flowy-net/build.rs +++ b/frontend/rust-lib/flowy-net/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-net", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/frontend/rust-lib/flowy-user/build.rs b/frontend/rust-lib/flowy-user/build.rs index b871a59257..ef2b12f93d 100644 --- a/frontend/rust-lib/flowy-user/build.rs +++ b/frontend/rust-lib/flowy-user/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-user", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/shared-lib/Cargo.lock b/shared-lib/Cargo.lock index e0c18f6b65..d5463b5006 100644 --- a/shared-lib/Cargo.lock +++ b/shared-lib/Cargo.lock @@ -450,9 +450,11 @@ dependencies = [ "log", "proc-macro2", "quote", + "serde_json", "syn", "tokio", "trybuild", + "walkdir", ] [[package]] diff --git a/shared-lib/error-code/build.rs b/shared-lib/error-code/build.rs index f7a0c00685..ef2b12f93d 100644 --- a/shared-lib/error-code/build.rs +++ b/shared-lib/error-code/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("error-code", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/shared-lib/flowy-collaboration/build.rs b/shared-lib/flowy-collaboration/build.rs index ac622a2900..ef2b12f93d 100644 --- a/shared-lib/flowy-collaboration/build.rs +++ b/shared-lib/flowy-collaboration/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-collaboration", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/shared-lib/flowy-derive/Cargo.toml b/shared-lib/flowy-derive/Cargo.toml index a43edf23dd..0623cd7428 100644 --- a/shared-lib/flowy-derive/Cargo.toml +++ b/shared-lib/flowy-derive/Cargo.toml @@ -20,7 +20,9 @@ proc-macro2 = "1.0" flowy-ast = { path = "../flowy-ast" } lazy_static = {version = "1.4.0"} dashmap = "4.0" -lib-infra = { path = "../lib-infra"} +lib-infra = { path = "../lib-infra", features = ["proto_gen"]} +serde_json = "1.0" +walkdir = "2.3.1" [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs b/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs deleted file mode 100644 index cd66e80f04..0000000000 --- a/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs +++ /dev/null @@ -1,122 +0,0 @@ -#![cfg_attr(rustfmt, rustfmt::skip)] -use lazy_static::lazy_static; -use dashmap::DashMap; - -lazy_static! { - static ref map: DashMap = DashMap::new(); -} - -pub enum TypeCategory { - Array, - Map, - Str, - Protobuf, - Bytes, - Enum, - Opt, - Primitive, -} -// auto generate, do not edit -pub fn category_from_str(type_str: &str) -> TypeCategory { - let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string(); - if !map.contains_key(&root_absolute_path) { - println!("😁{}", root_absolute_path); - map.insert(root_absolute_path, "123".to_string()); - - } - - match type_str { - "Vec" => TypeCategory::Array, - "HashMap" => TypeCategory::Map, - "u8" => TypeCategory::Bytes, - "String" => TypeCategory::Str, - "FFIRequest" - | "FFIResponse" - | "FlowyError" - | "SubscribeObject" - | "NetworkState" - | "SignInRequest" - | "SignInParams" - | "SignInResponse" - | "SignUpRequest" - | "SignUpParams" - | "SignUpResponse" - | "UserToken" - | "UserProfile" - | "UpdateUserRequest" - | "UpdateUserParams" - | "UserPreferences" - | "AppearanceSettings" - | "LocaleSettings" - | "ClientRevisionWSData" - | "ServerRevisionWSData" - | "NewDocumentUser" - | "FolderInfo" - | "Revision" - | "RepeatedRevision" - | "RevId" - | "RevisionRange" - | "CreateDocParams" - | "DocumentInfo" - | "ResetDocumentParams" - | "DocumentDelta" - | "NewDocUser" - | "DocumentId" - | "WSError" - | "WebSocketRawMessage" - | "Workspace" - | "RepeatedWorkspace" - | "CreateWorkspaceRequest" - | "CreateWorkspaceParams" - | "QueryWorkspaceRequest" - | "WorkspaceId" - | "CurrentWorkspaceSetting" - | "UpdateWorkspaceRequest" - | "UpdateWorkspaceParams" - | "ExportRequest" - | "ExportData" - | "App" - | "RepeatedApp" - | "CreateAppRequest" - | "ColorStyle" - | "CreateAppParams" - | "QueryAppRequest" - | "AppId" - | "UpdateAppRequest" - | "UpdateAppParams" - | "Trash" - | "RepeatedTrash" - | "RepeatedTrashId" - | "TrashId" - | "View" - | "RepeatedView" - | "CreateViewRequest" - | "CreateViewParams" - | "QueryViewRequest" - | "ViewId" - | "RepeatedViewId" - | "UpdateViewRequest" - | "UpdateViewParams" - => TypeCategory::Protobuf, - "FFIStatusCode" - | "FolderEvent" - | "FolderNotification" - | "NetworkEvent" - | "NetworkType" - | "UserEvent" - | "UserNotification" - | "ClientRevisionWSDataType" - | "ServerRevisionWSDataType" - | "RevisionState" - | "RevType" - | "ErrorCode" - | "WSChannel" - | "ExportType" - | "TrashType" - | "ViewType" - => TypeCategory::Enum, - - "Option" => TypeCategory::Opt, - _ => TypeCategory::Primitive, - } -} diff --git a/shared-lib/flowy-derive/src/derive_cache/mod.rs b/shared-lib/flowy-derive/src/derive_cache/mod.rs deleted file mode 100644 index 1b48d15980..0000000000 --- a/shared-lib/flowy-derive/src/derive_cache/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -#![allow(clippy::module_inception)] -mod derive_cache; - -pub use derive_cache::*; diff --git a/shared-lib/flowy-derive/src/lib.rs b/shared-lib/flowy-derive/src/lib.rs index 82f0002fb3..71ff82b5ad 100644 --- a/shared-lib/flowy-derive/src/lib.rs +++ b/shared-lib/flowy-derive/src/lib.rs @@ -9,7 +9,6 @@ use syn::{parse_macro_input, DeriveInput}; extern crate quote; mod dart_event; -mod derive_cache; mod proto_buf; // Inspired by https://serde.rs/attributes.html diff --git a/shared-lib/flowy-derive/src/proto_buf/deserialize.rs b/shared-lib/flowy-derive/src/proto_buf/deserialize.rs index 80b37e3dcd..af4b7f7345 100644 --- a/shared-lib/flowy-derive/src/proto_buf/deserialize.rs +++ b/shared-lib/flowy-derive/src/proto_buf/deserialize.rs @@ -1,4 +1,4 @@ -use crate::{derive_cache::TypeCategory, proto_buf::util::*}; +use crate::proto_buf::util::*; use flowy_ast::*; use proc_macro2::{Span, TokenStream}; diff --git a/shared-lib/flowy-derive/src/proto_buf/serialize.rs b/shared-lib/flowy-derive/src/proto_buf/serialize.rs index 7d1ce44a45..d0162c90d3 100644 --- a/shared-lib/flowy-derive/src/proto_buf/serialize.rs +++ b/shared-lib/flowy-derive/src/proto_buf/serialize.rs @@ -1,8 +1,5 @@ #![allow(clippy::while_let_on_iterator)] -use crate::{ - derive_cache::TypeCategory, - proto_buf::util::{get_member_ident, ident_category}, -}; +use crate::proto_buf::util::{get_member_ident, ident_category, TypeCategory}; use flowy_ast::*; use proc_macro2::TokenStream; diff --git a/shared-lib/flowy-derive/src/proto_buf/util.rs b/shared-lib/flowy-derive/src/proto_buf/util.rs index aec13e7358..7260d97844 100644 --- a/shared-lib/flowy-derive/src/proto_buf/util.rs +++ b/shared-lib/flowy-derive/src/proto_buf/util.rs @@ -1,8 +1,14 @@ -use crate::derive_cache::*; +use dashmap::{DashMap, DashSet}; use flowy_ast::{Ctxt, TyInfo}; +use lazy_static::lazy_static; +use lib_infra::proto_gen::ProtoCache; +use std::fs::File; +use std::io::Read; +use std::sync::atomic::{AtomicBool, Ordering}; +use walkdir::WalkDir; pub fn ident_category(ident: &syn::Ident) -> TypeCategory { - let ident_str: &str = &ident.to_string(); + let ident_str = ident.to_string(); category_from_str(ident_str) } @@ -20,3 +26,84 @@ pub fn assert_bracket_ty_is_some(ctxt: &Ctxt, ty_info: &TyInfo) { ctxt.error_spanned_by(ty_info.ty, "Invalid bracketed type when gen de token steam".to_string()); } } + +lazy_static! { + static ref READ_FLAG: DashSet = DashSet::new(); + static ref CACHE_INFO: DashMap> = DashMap::new(); + static ref IS_LOAD: AtomicBool = AtomicBool::new(false); +} + +#[derive(Eq, Hash, PartialEq)] +pub enum TypeCategory { + Array, + Map, + Str, + Protobuf, + Bytes, + Enum, + Opt, + Primitive, +} +// auto generate, do not edit +pub fn category_from_str(type_str: String) -> TypeCategory { + if !IS_LOAD.load(Ordering::SeqCst) { + IS_LOAD.store(true, Ordering::SeqCst); + // Dependents on another crate file is not good, just leave it here. + // Maybe find another way to read the .cache in the future. + let cache_dir = format!("{}/../lib-infra/.cache", env!("CARGO_MANIFEST_DIR")); + for path in WalkDir::new(cache_dir) + .into_iter() + .filter_map(|e| e.ok()) + .map(|e| e.path().to_str().unwrap().to_string()) + { + match read_file(&path) { + None => {} + Some(s) => { + let cache: ProtoCache = serde_json::from_str(&s).unwrap(); + CACHE_INFO + .entry(TypeCategory::Protobuf) + .or_insert(vec![]) + .extend(cache.structs); + CACHE_INFO + .entry(TypeCategory::Enum) + .or_insert(vec![]) + .extend(cache.enums); + } + } + } + } + + if let Some(protobuf) = CACHE_INFO.get(&TypeCategory::Protobuf) { + if protobuf.contains(&type_str) { + return TypeCategory::Protobuf; + } + } + + if let Some(protobuf) = CACHE_INFO.get(&TypeCategory::Enum) { + if protobuf.contains(&type_str) { + return TypeCategory::Enum; + } + } + + match type_str.as_str() { + "Vec" => TypeCategory::Array, + "HashMap" => TypeCategory::Map, + "u8" => TypeCategory::Bytes, + "String" => TypeCategory::Str, + "Option" => TypeCategory::Opt, + _ => TypeCategory::Primitive, + } +} + +fn read_file(path: &str) -> Option { + match File::open(path) { + Ok(mut file) => { + let mut content = String::new(); + match file.read_to_string(&mut content) { + Ok(_) => Some(content), + Err(_) => None, + } + } + Err(_) => None, + } +} diff --git a/shared-lib/flowy-folder-data-model/build.rs b/shared-lib/flowy-folder-data-model/build.rs index 6691d2db50..ef2b12f93d 100644 --- a/shared-lib/flowy-folder-data-model/build.rs +++ b/shared-lib/flowy-folder-data-model/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-folder-data-model", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/shared-lib/flowy-user-data-model/build.rs b/shared-lib/flowy-user-data-model/build.rs index e2263ed202..ef2b12f93d 100644 --- a/shared-lib/flowy-user-data-model/build.rs +++ b/shared-lib/flowy-user-data-model/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("flowy-user-data-model", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); } diff --git a/shared-lib/lib-infra/src/lib.rs b/shared-lib/lib-infra/src/lib.rs index e60ee026e5..fbe9b11486 100644 --- a/shared-lib/lib-infra/src/lib.rs +++ b/shared-lib/lib-infra/src/lib.rs @@ -5,7 +5,7 @@ pub mod retry; pub mod pb; #[cfg(feature = "proto_gen")] -mod proto_gen; +pub mod proto_gen; #[allow(dead_code)] pub fn uuid_string() -> String { diff --git a/shared-lib/lib-infra/src/pb.rs b/shared-lib/lib-infra/src/pb.rs index 094a22ccb6..1b0f3aa70f 100644 --- a/shared-lib/lib-infra/src/pb.rs +++ b/shared-lib/lib-infra/src/pb.rs @@ -10,9 +10,9 @@ use std::io::Write; use std::process::Command; use walkdir::WalkDir; -pub fn gen_files(name: &str, root: &str) { +pub fn gen_files(crate_name: &str, root: &str) { #[cfg(feature = "proto_gen")] - let _ = gen_protos(); + let _ = gen_protos(crate_name); let mut paths = vec![]; let mut file_names = vec![]; @@ -31,7 +31,7 @@ pub fn gen_files(name: &str, root: &str) { } println!("cargo:rerun-if-changed=build.rs"); #[cfg(feature = "dart")] - gen_pb_for_dart(name, root, &paths, &file_names); + gen_pb_for_dart(crate_name, root, &paths, &file_names); protoc_rust::Codegen::new() .out_dir("./src/protobuf/model") @@ -120,9 +120,10 @@ fn run_command(cmd: &str) -> bool { } #[cfg(feature = "proto_gen")] -fn gen_protos() -> Vec { +fn gen_protos(crate_name: &str) -> Vec { + let cache_path = env!("CARGO_MANIFEST_DIR"); let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string(); - let crate_context = ProtoGenerator::gen(&root); + let crate_context = ProtoGenerator::gen(crate_name, &root, cache_path); let proto_crates = crate_context .iter() .map(|info| info.protobuf_crate.clone()) diff --git a/shared-lib/lib-infra/src/proto_gen/proto_gen.rs b/shared-lib/lib-infra/src/proto_gen/proto_gen.rs index c9d454481c..6979ca9492 100644 --- a/shared-lib/lib-infra/src/proto_gen/proto_gen.rs +++ b/shared-lib/lib-infra/src/proto_gen/proto_gen.rs @@ -4,12 +4,13 @@ use crate::proto_gen::template::write_derive_meta; use crate::proto_gen::util::*; use crate::proto_gen::ProtoFile; use std::fs::File; +use std::path::Path; use std::{fs::OpenOptions, io::Write}; pub(crate) struct ProtoGenerator(); impl ProtoGenerator { - pub(crate) fn gen(root: &str) -> Vec { - let crate_contexts = parse_crate_protobuf(vec![root.to_owned()]); + pub(crate) fn gen(crate_name: &str, crate_path: &str, cache_path: &str) -> Vec { + let crate_contexts = parse_crate_protobuf(vec![crate_path.to_owned()]); write_proto_files(&crate_contexts); write_rust_crate_mod_file(&crate_contexts); for crate_info in &crate_contexts { @@ -20,20 +21,25 @@ impl ProtoGenerator { let cache = ProtoCache::from_crate_contexts(&crate_contexts); let cache_str = serde_json::to_string(&cache).unwrap(); - let protobuf_dart = format!("{}/.proto_cache", root); + let cache_dir = format!("{}/.cache/{}", cache_path, crate_name); + if !Path::new(&cache_dir).exists() { + std::fs::create_dir_all(&cache_dir).unwrap(); + } + + let protobuf_cache_path = format!("{}/proto_cache", cache_dir); match std::fs::OpenOptions::new() .create(true) .write(true) .append(false) .truncate(true) - .open(&protobuf_dart) + .open(&protobuf_cache_path) { Ok(ref mut file) => { file.write_all(cache_str.as_bytes()).unwrap(); File::flush(file).unwrap(); } Err(err) => { - panic!("Failed to open file: {}", err); + panic!("Failed to open file: {}", protobuf_cache_path); } } @@ -83,7 +89,7 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) { } } -#[derive(serde::Serialize)] +#[derive(serde::Serialize, serde::Deserialize)] pub struct ProtoCache { pub structs: Vec, pub enums: Vec, diff --git a/shared-lib/lib-infra/src/proto_gen/template/derive_meta/derive_meta.rs b/shared-lib/lib-infra/src/proto_gen/template/derive_meta/derive_meta.rs index 7b29825551..f84ebef5f7 100644 --- a/shared-lib/lib-infra/src/proto_gen/template/derive_meta/derive_meta.rs +++ b/shared-lib/lib-infra/src/proto_gen/template/derive_meta/derive_meta.rs @@ -26,7 +26,7 @@ impl ProtobufDeriveMeta { self.context.insert("names", &self.structs); self.context.insert("enums", &self.enums); - let tera = get_tera("proto_gen/template/derive_meta"); + let tera = get_tera("derive_meta"); match tera.render("derive_meta.tera", &self.context) { Ok(r) => Some(r), Err(e) => { diff --git a/shared-lib/lib-infra/src/proto_gen/template/mod.rs b/shared-lib/lib-infra/src/proto_gen/template/mod.rs index 63fbbab5f9..27624e66a2 100644 --- a/shared-lib/lib-infra/src/proto_gen/template/mod.rs +++ b/shared-lib/lib-infra/src/proto_gen/template/mod.rs @@ -8,7 +8,7 @@ use std::io::Read; use tera::Tera; pub fn get_tera(directory: &str) -> Tera { - let mut root = format!("{}/../shared-lib/lib-infra/src/", env!("CARGO_MAKE_WORKING_DIRECTORY")); + let mut root = format!("{}/../", file!()); root.push_str(directory); let root_absolute_path = std::fs::canonicalize(root).unwrap().as_path().display().to_string(); diff --git a/shared-lib/lib-infra/src/proto_gen/template/proto_file/enum_template.rs b/shared-lib/lib-infra/src/proto_gen/template/proto_file/enum_template.rs index 52a5d906a6..404b3eb88b 100644 --- a/shared-lib/lib-infra/src/proto_gen/template/proto_file/enum_template.rs +++ b/shared-lib/lib-infra/src/proto_gen/template/proto_file/enum_template.rs @@ -26,7 +26,7 @@ impl EnumTemplate { pub fn render(&mut self) -> Option { self.context.insert("items", &self.items); - let tera = get_tera("proto_gen/template/proto_file"); + let tera = get_tera("proto_file"); match tera.render("enum.tera", &self.context) { Ok(r) => Some(r), Err(e) => { diff --git a/shared-lib/lib-infra/src/proto_gen/template/proto_file/struct_template.rs b/shared-lib/lib-infra/src/proto_gen/template/proto_file/struct_template.rs index 43f8835e5f..d7c0568561 100644 --- a/shared-lib/lib-infra/src/proto_gen/template/proto_file/struct_template.rs +++ b/shared-lib/lib-infra/src/proto_gen/template/proto_file/struct_template.rs @@ -95,7 +95,7 @@ impl StructTemplate { pub fn render(&mut self) -> Option { self.context.insert("fields", &self.fields); - let tera = get_tera("proto_gen/template/proto_file"); + let tera = get_tera("proto_file"); match tera.render("struct.tera", &self.context) { Ok(r) => Some(r), Err(e) => { diff --git a/shared-lib/lib-ws/build.rs b/shared-lib/lib-ws/build.rs index 2bdb72db52..ef2b12f93d 100644 --- a/shared-lib/lib-ws/build.rs +++ b/shared-lib/lib-ws/build.rs @@ -1,5 +1,5 @@ use lib_infra::pb; fn main() { - pb::gen_files("lib-ws", "./src/protobuf/proto"); + pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto"); }