From 1c3c519b1462aed90fe378ead82cec0a28eceec7 Mon Sep 17 00:00:00 2001 From: appflowy Date: Wed, 9 Feb 2022 13:08:49 +0800 Subject: [PATCH] refactor: cache proto file info --- .gitignore | 1 + frontend/rust-lib/Cargo.lock | 4 + frontend/rust-lib/flowy-folder/Cargo.toml | 2 +- frontend/rust-lib/flowy-sdk/Cargo.toml | 3 - frontend/rust-lib/flowy-sdk/build.rs | 9 -- shared-lib/Cargo.lock | 4 + shared-lib/flowy-derive/Cargo.toml | 3 + .../src/derive_cache/derive_cache.rs | 14 +++ shared-lib/lib-infra/Cargo.toml | 9 +- shared-lib/lib-infra/src/lib.rs | 2 +- shared-lib/lib-infra/src/pb.rs | 99 +++++++++---------- shared-lib/lib-infra/src/proto_gen/ast.rs | 2 +- shared-lib/lib-infra/src/proto_gen/mod.rs | 5 +- .../lib-infra/src/proto_gen/proto_gen.rs | 41 ++++++++ 14 files changed, 125 insertions(+), 73 deletions(-) delete mode 100644 frontend/rust-lib/flowy-sdk/build.rs diff --git a/.gitignore b/.gitignore index 517b5e2f75..7f594b1012 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ package-lock.json yarn.lock node_modules +**/.proto_cache \ No newline at end of file diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 8bd18cff13..0906fe5ede 100755 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -900,7 +900,10 @@ dependencies = [ name = "flowy-derive" version = "0.1.0" dependencies = [ + "dashmap", "flowy-ast", + "lazy_static", + "lib-infra", "proc-macro2", "quote", "syn", @@ -1723,6 +1726,7 @@ dependencies = [ "protoc-rust", "rand 0.8.4", "serde", + "serde_json", "similar", "syn", "tera", diff --git a/frontend/rust-lib/flowy-folder/Cargo.toml b/frontend/rust-lib/flowy-folder/Cargo.toml index 43d259dd4b..f51c3fb4eb 100644 --- a/frontend/rust-lib/flowy-folder/Cargo.toml +++ b/frontend/rust-lib/flowy-folder/Cargo.toml @@ -56,4 +56,4 @@ flowy_unit_test = ["lib-ot/flowy_unit_test", "flowy-sync/flowy_unit_test"] dart = ["lib-infra/dart", "flowy-folder/dart", "flowy-folder/dart",] [build-dependencies] -lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen"] } \ No newline at end of file +lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen", "proto_gen"] } \ No newline at end of file diff --git a/frontend/rust-lib/flowy-sdk/Cargo.toml b/frontend/rust-lib/flowy-sdk/Cargo.toml index 7b50c8ff27..b303f89b30 100644 --- a/frontend/rust-lib/flowy-sdk/Cargo.toml +++ b/frontend/rust-lib/flowy-sdk/Cargo.toml @@ -40,6 +40,3 @@ futures-util = "0.3.15" http_server = ["flowy-user/http_server", "flowy-folder/http_server", "flowy-document/http_server"] use_bunyan = ["lib-log/use_bunyan"] dart = ["flowy-user/dart", "flowy-net/dart", "flowy-folder/dart", "flowy-collaboration/dart"] - -[build-dependencies] -lib-infra = { path = "../../../shared-lib/lib-infra", features = ["proto_gen"] } \ No newline at end of file diff --git a/frontend/rust-lib/flowy-sdk/build.rs b/frontend/rust-lib/flowy-sdk/build.rs deleted file mode 100644 index 3de6d6ddbc..0000000000 --- a/frontend/rust-lib/flowy-sdk/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -// use lib_infra::proto_gen; - -fn main() { - // let derive_meta_dir = format!( - // "{}/{}", - // env!("CARGO_MAKE_WORKING_DIRECTORY"), - // env!("PROTOBUF_DERIVE_CACHE"), - // ); -} diff --git a/shared-lib/Cargo.lock b/shared-lib/Cargo.lock index 6b61811e77..e0c18f6b65 100644 --- a/shared-lib/Cargo.lock +++ b/shared-lib/Cargo.lock @@ -443,7 +443,10 @@ dependencies = [ name = "flowy-derive" version = "0.1.0" dependencies = [ + "dashmap", "flowy-ast", + "lazy_static", + "lib-infra", "log", "proc-macro2", "quote", @@ -802,6 +805,7 @@ dependencies = [ "protoc-rust", "rand 0.8.4", "serde", + "serde_json", "similar", "syn", "tera", diff --git a/shared-lib/flowy-derive/Cargo.toml b/shared-lib/flowy-derive/Cargo.toml index 0b7cc365e7..a43edf23dd 100644 --- a/shared-lib/flowy-derive/Cargo.toml +++ b/shared-lib/flowy-derive/Cargo.toml @@ -18,6 +18,9 @@ syn = { version = "1.0.60", features = ["extra-traits", "visit"] } quote = "1.0" proc-macro2 = "1.0" flowy-ast = { path = "../flowy-ast" } +lazy_static = {version = "1.4.0"} +dashmap = "4.0" +lib-infra = { path = "../lib-infra"} [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs b/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs index ad669eeed4..cd66e80f04 100644 --- a/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs +++ b/shared-lib/flowy-derive/src/derive_cache/derive_cache.rs @@ -1,4 +1,11 @@ #![cfg_attr(rustfmt, rustfmt::skip)] +use lazy_static::lazy_static; +use dashmap::DashMap; + +lazy_static! { + static ref map: DashMap = DashMap::new(); +} + pub enum TypeCategory { Array, Map, @@ -11,6 +18,13 @@ pub enum TypeCategory { } // auto generate, do not edit pub fn category_from_str(type_str: &str) -> TypeCategory { + let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string(); + if !map.contains_key(&root_absolute_path) { + println!("😁{}", root_absolute_path); + map.insert(root_absolute_path, "123".to_string()); + + } + match type_str { "Vec" => TypeCategory::Array, "HashMap" => TypeCategory::Map, diff --git a/shared-lib/lib-infra/Cargo.toml b/shared-lib/lib-infra/Cargo.toml index 5415b42dca..e95cf3f5df 100644 --- a/shared-lib/lib-infra/Cargo.toml +++ b/shared-lib/lib-infra/Cargo.toml @@ -14,6 +14,8 @@ pin-project = "1.0" futures-core = { version = "0.3" } tokio = { version = "1.0", features = ["time", "rt"] } rand = "0.8.3" +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" cmd_lib = { version = "1", optional = true } protoc-rust = { version = "2", optional = true } @@ -28,11 +30,10 @@ tera = { version = "1.5.0", optional = true} itertools = {versino = "0.10", optional = true} phf = { version = "0.8.0", features = ["macros"], optional = true } console = {version = "0.14.0", optional = true} -serde = { version = "1.0", features = ["derive"], optional = true} + toml = {version = "0.5.8", optional = true} [features] -pb_gen = ["cmd_lib", "protoc-rust", "walkdir"] proto_gen = [ "flowy-ast", "similar", @@ -44,7 +45,7 @@ proto_gen = [ "phf", "walkdir", "console", - "serde", "toml" ] -dart = [] \ No newline at end of file +pb_gen = ["cmd_lib", "protoc-rust", "walkdir"] +dart = ["proto_gen"] \ No newline at end of file diff --git a/shared-lib/lib-infra/src/lib.rs b/shared-lib/lib-infra/src/lib.rs index fbe9b11486..e60ee026e5 100644 --- a/shared-lib/lib-infra/src/lib.rs +++ b/shared-lib/lib-infra/src/lib.rs @@ -5,7 +5,7 @@ pub mod retry; pub mod pb; #[cfg(feature = "proto_gen")] -pub mod proto_gen; +mod proto_gen; #[allow(dead_code)] pub fn uuid_string() -> String { diff --git a/shared-lib/lib-infra/src/pb.rs b/shared-lib/lib-infra/src/pb.rs index 32e9377804..094a22ccb6 100644 --- a/shared-lib/lib-infra/src/pb.rs +++ b/shared-lib/lib-infra/src/pb.rs @@ -1,69 +1,44 @@ -#![allow(clippy::all)] #![allow(unused_imports)] #![allow(unused_attributes)] #![allow(dead_code)] -use crate::proto_gen; -use crate::proto_gen::proto_info::ProtobufCrate; -use crate::proto_gen::util::suffix_relative_to_path; -use crate::proto_gen::ProtoGenerator; + +#[cfg(feature = "proto_gen")] +use crate::proto_gen::*; use log::info; use std::fs::File; use std::io::Write; use std::process::Command; use walkdir::WalkDir; -fn gen_protos(root: &str) -> Vec { - let crate_context = ProtoGenerator::gen(root); - let proto_crates = crate_context - .iter() - .map(|info| info.protobuf_crate.clone()) - .collect::>(); - crate_context - .into_iter() - .map(|info| info.files) - .flatten() - .for_each(|file| { - println!("cargo:rerun-if-changed={}", file.file_path); - }); - proto_crates -} - pub fn gen_files(name: &str, root: &str) { - let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string(); - for protobuf_crate in gen_protos(&root_absolute_path) { - let mut paths = vec![]; - let mut file_names = vec![]; + #[cfg(feature = "proto_gen")] + let _ = gen_protos(); - let proto_relative_path = suffix_relative_to_path(&protobuf_crate.proto_output_dir(), &root_absolute_path); + let mut paths = vec![]; + let mut file_names = vec![]; - for (path, file_name) in WalkDir::new(proto_relative_path) - .into_iter() - .filter_map(|e| e.ok()) - .map(|e| { - let path = e.path().to_str().unwrap().to_string(); - let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string(); - (path, file_name) - }) - { - if path.ends_with(".proto") { - // https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project - println!("cargo:rerun-if-changed={}", path); - paths.push(path); - file_names.push(file_name); - } + for (path, file_name) in WalkDir::new(root).into_iter().filter_map(|e| e.ok()).map(|e| { + let path = e.path().to_str().unwrap().to_string(); + let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string(); + (path, file_name) + }) { + if path.ends_with(".proto") { + // https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project + println!("cargo:rerun-if-changed={}", path); + paths.push(path); + file_names.push(file_name); } - println!("cargo:rerun-if-changed=build.rs"); - - #[cfg(feature = "dart")] - gen_pb_for_dart(name, root, &paths, &file_names); - - protoc_rust::Codegen::new() - .out_dir("./src/protobuf/model") - .inputs(&paths) - .include(root) - .run() - .expect("Running protoc failed."); } + println!("cargo:rerun-if-changed=build.rs"); + #[cfg(feature = "dart")] + gen_pb_for_dart(name, root, &paths, &file_names); + + protoc_rust::Codegen::new() + .out_dir("./src/protobuf/model") + .inputs(&paths) + .include(root) + .run() + .expect("Running protoc failed."); } #[cfg(feature = "dart")] @@ -143,3 +118,23 @@ fn run_command(cmd: &str) -> bool { }; output.success() } + +#[cfg(feature = "proto_gen")] +fn gen_protos() -> Vec { + let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string(); + let crate_context = ProtoGenerator::gen(&root); + let proto_crates = crate_context + .iter() + .map(|info| info.protobuf_crate.clone()) + .collect::>(); + + crate_context + .into_iter() + .map(|info| info.files) + .flatten() + .for_each(|file| { + println!("cargo:rerun-if-changed={}", file.file_path); + }); + + proto_crates +} diff --git a/shared-lib/lib-infra/src/proto_gen/ast.rs b/shared-lib/lib-infra/src/proto_gen/ast.rs index 898987339c..2c313c0dea 100644 --- a/shared-lib/lib-infra/src/proto_gen/ast.rs +++ b/shared-lib/lib-infra/src/proto_gen/ast.rs @@ -1,6 +1,6 @@ -use crate::proto_gen::proto_info::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext}; use crate::proto_gen::template::{EnumTemplate, StructTemplate}; use crate::proto_gen::util::*; +use crate::proto_gen::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext}; use fancy_regex::Regex; use flowy_ast::*; use lazy_static::lazy_static; diff --git a/shared-lib/lib-infra/src/proto_gen/mod.rs b/shared-lib/lib-infra/src/proto_gen/mod.rs index 0a80708bbf..1bf7e159de 100644 --- a/shared-lib/lib-infra/src/proto_gen/mod.rs +++ b/shared-lib/lib-infra/src/proto_gen/mod.rs @@ -1,8 +1,9 @@ mod ast; mod flowy_toml; mod proto_gen; -pub mod proto_info; +mod proto_info; mod template; pub mod util; -pub(crate) use proto_gen::*; +pub use proto_gen::*; +pub use proto_info::*; diff --git a/shared-lib/lib-infra/src/proto_gen/proto_gen.rs b/shared-lib/lib-infra/src/proto_gen/proto_gen.rs index fd25332d53..c9d454481c 100644 --- a/shared-lib/lib-infra/src/proto_gen/proto_gen.rs +++ b/shared-lib/lib-infra/src/proto_gen/proto_gen.rs @@ -2,6 +2,8 @@ use crate::proto_gen::ast::parse_crate_protobuf; use crate::proto_gen::proto_info::ProtobufCrateContext; use crate::proto_gen::template::write_derive_meta; use crate::proto_gen::util::*; +use crate::proto_gen::ProtoFile; +use std::fs::File; use std::{fs::OpenOptions, io::Write}; pub(crate) struct ProtoGenerator(); @@ -16,6 +18,25 @@ impl ProtoGenerator { crate_info.create_crate_mod_file(); } + let cache = ProtoCache::from_crate_contexts(&crate_contexts); + let cache_str = serde_json::to_string(&cache).unwrap(); + let protobuf_dart = format!("{}/.proto_cache", root); + match std::fs::OpenOptions::new() + .create(true) + .write(true) + .append(false) + .truncate(true) + .open(&protobuf_dart) + { + Ok(ref mut file) => { + file.write_all(cache_str.as_bytes()).unwrap(); + File::flush(file).unwrap(); + } + Err(err) => { + panic!("Failed to open file: {}", err); + } + } + crate_contexts } } @@ -61,3 +82,23 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) { } } } + +#[derive(serde::Serialize)] +pub struct ProtoCache { + pub structs: Vec, + pub enums: Vec, +} + +impl ProtoCache { + fn from_crate_contexts(crate_contexts: &[ProtobufCrateContext]) -> Self { + let proto_files = crate_contexts + .iter() + .map(|ref crate_info| &crate_info.files) + .flatten() + .collect::>(); + + let structs: Vec = proto_files.iter().map(|info| info.structs.clone()).flatten().collect(); + let enums: Vec = proto_files.iter().map(|info| info.enums.clone()).flatten().collect(); + Self { structs, enums } + } +}