Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: cache proto file info

Browse Source
This commit is contained in:
appflowy 2022-02-09 13:08:49 +08:00
parent 7646f984e0
commit 1c3c519b14
14 changed files with 125 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -15,3 +15,4 @@
package-lock.json
yarn.lock
node_modules
**/.proto_cache

4
frontend/rust-lib/Cargo.lock generated
View File

@ -900,7 +900,10 @@ dependencies = [
name = "flowy-derive"
version = "0.1.0"
dependencies = [
"dashmap",
"flowy-ast",
"lazy_static",
"lib-infra",
"proc-macro2",
"quote",
"syn",
@ -1723,6 +1726,7 @@ dependencies = [
"protoc-rust",
"rand 0.8.4",
"serde",
"serde_json",
"similar",
"syn",
"tera",

2
frontend/rust-lib/flowy-folder/Cargo.toml
View File

@ -56,4 +56,4 @@ flowy_unit_test = ["lib-ot/flowy_unit_test", "flowy-sync/flowy_unit_test"]
dart = ["lib-infra/dart", "flowy-folder/dart", "flowy-folder/dart",]
[build-dependencies]
lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen"] }
lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen", "proto_gen"] }

3
frontend/rust-lib/flowy-sdk/Cargo.toml
View File

@ -40,6 +40,3 @@ futures-util = "0.3.15"
http_server = ["flowy-user/http_server", "flowy-folder/http_server", "flowy-document/http_server"]
use_bunyan = ["lib-log/use_bunyan"]
dart = ["flowy-user/dart", "flowy-net/dart", "flowy-folder/dart", "flowy-collaboration/dart"]
[build-dependencies]
lib-infra = { path = "../../../shared-lib/lib-infra", features = ["proto_gen"] }

9
frontend/rust-lib/flowy-sdk/build.rs
View File

@ -1,9 +0,0 @@
// use lib_infra::proto_gen;
fn main() {
// let derive_meta_dir = format!(
// "{}/{}",
// env!("CARGO_MAKE_WORKING_DIRECTORY"),
// env!("PROTOBUF_DERIVE_CACHE"),
// );
}

4
shared-lib/Cargo.lock generated
View File

@ -443,7 +443,10 @@ dependencies = [
name = "flowy-derive"
version = "0.1.0"
dependencies = [
"dashmap",
"flowy-ast",
"lazy_static",
"lib-infra",
"log",
"proc-macro2",
"quote",
@ -802,6 +805,7 @@ dependencies = [
"protoc-rust",
"rand 0.8.4",
"serde",
"serde_json",
"similar",
"syn",
"tera",

3
shared-lib/flowy-derive/Cargo.toml
View File

@ -18,6 +18,9 @@ syn = { version = "1.0.60", features = ["extra-traits", "visit"] }
quote = "1.0"
proc-macro2 = "1.0"
flowy-ast = { path = "../flowy-ast" }
lazy_static = {version = "1.4.0"}
dashmap = "4.0"
lib-infra = { path = "../lib-infra"}
[dev-dependencies]
tokio = { version = "1", features = ["full"] }

14
shared-lib/flowy-derive/src/derive_cache/derive_cache.rs
View File

@ -1,4 +1,11 @@
#![cfg_attr(rustfmt, rustfmt::skip)]
use lazy_static::lazy_static;
use dashmap::DashMap;
lazy_static! {
static ref map: DashMap<String,String> = DashMap::new();
}
pub enum TypeCategory {
Array,
Map,
@ -11,6 +18,13 @@ pub enum TypeCategory {
}
// auto generate, do not edit
pub fn category_from_str(type_str: &str) -> TypeCategory {
let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
if !map.contains_key(&root_absolute_path) {
println!("😁{}", root_absolute_path);
map.insert(root_absolute_path, "123".to_string());
}
match type_str {
"Vec" => TypeCategory::Array,
"HashMap" => TypeCategory::Map,

9
shared-lib/lib-infra/Cargo.toml
View File

@ -14,6 +14,8 @@ pin-project = "1.0"
futures-core = { version = "0.3" }
tokio = { version = "1.0", features = ["time", "rt"] }
rand = "0.8.3"
serde = { version = "1.0", features = ["derive"]}
serde_json = "1.0"
cmd_lib = { version = "1", optional = true }
protoc-rust = { version = "2", optional = true }
@ -28,11 +30,10 @@ tera = { version = "1.5.0", optional = true}
itertools = {versino = "0.10", optional = true}
phf = { version = "0.8.0", features = ["macros"], optional = true }
console = {version = "0.14.0", optional = true}
serde = { version = "1.0", features = ["derive"], optional = true}
toml = {version = "0.5.8", optional = true}
[features]
pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
proto_gen = [
"flowy-ast",
"similar",
@ -44,7 +45,7 @@ proto_gen = [
"phf",
"walkdir",
"console",
"serde",
"toml"
]
dart = []
pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
dart = ["proto_gen"]

2
shared-lib/lib-infra/src/lib.rs
View File

@ -5,7 +5,7 @@ pub mod retry;
pub mod pb;
#[cfg(feature = "proto_gen")]
pub mod proto_gen;
mod proto_gen;
#[allow(dead_code)]
pub fn uuid_string() -> String {

99
shared-lib/lib-infra/src/pb.rs
View File

@ -1,69 +1,44 @@
#![allow(clippy::all)]
#![allow(unused_imports)]
#![allow(unused_attributes)]
#![allow(dead_code)]
use crate::proto_gen;
use crate::proto_gen::proto_info::ProtobufCrate;
use crate::proto_gen::util::suffix_relative_to_path;
use crate::proto_gen::ProtoGenerator;
#[cfg(feature = "proto_gen")]
use crate::proto_gen::*;
use log::info;
use std::fs::File;
use std::io::Write;
use std::process::Command;
use walkdir::WalkDir;
fn gen_protos(root: &str) -> Vec<ProtobufCrate> {
let crate_context = ProtoGenerator::gen(root);
let proto_crates = crate_context
.iter()
.map(|info| info.protobuf_crate.clone())
.collect::<Vec<_>>();
crate_context
.into_iter()
.map(|info| info.files)
.flatten()
.for_each(|file| {
println!("cargo:rerun-if-changed={}", file.file_path);
});
proto_crates
}
pub fn gen_files(name: &str, root: &str) {
let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
for protobuf_crate in gen_protos(&root_absolute_path) {
let mut paths = vec![];
let mut file_names = vec![];
#[cfg(feature = "proto_gen")]
let _ = gen_protos();
let proto_relative_path = suffix_relative_to_path(&protobuf_crate.proto_output_dir(), &root_absolute_path);
let mut paths = vec![];
let mut file_names = vec![];
for (path, file_name) in WalkDir::new(proto_relative_path)
.into_iter()
.filter_map(|e| e.ok())
.map(|e| {
let path = e.path().to_str().unwrap().to_string();
let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string();
(path, file_name)
})
{
if path.ends_with(".proto") {
// https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project
println!("cargo:rerun-if-changed={}", path);
paths.push(path);
file_names.push(file_name);
}
for (path, file_name) in WalkDir::new(root).into_iter().filter_map(|e| e.ok()).map(|e| {
let path = e.path().to_str().unwrap().to_string();
let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string();
(path, file_name)
}) {
if path.ends_with(".proto") {
// https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project
println!("cargo:rerun-if-changed={}", path);
paths.push(path);
file_names.push(file_name);
}
println!("cargo:rerun-if-changed=build.rs");
#[cfg(feature = "dart")]
gen_pb_for_dart(name, root, &paths, &file_names);
protoc_rust::Codegen::new()
.out_dir("./src/protobuf/model")
.inputs(&paths)
.include(root)
.run()
.expect("Running protoc failed.");
}
println!("cargo:rerun-if-changed=build.rs");
#[cfg(feature = "dart")]
gen_pb_for_dart(name, root, &paths, &file_names);
protoc_rust::Codegen::new()
.out_dir("./src/protobuf/model")
.inputs(&paths)
.include(root)
.run()
.expect("Running protoc failed.");
}
#[cfg(feature = "dart")]
@ -143,3 +118,23 @@ fn run_command(cmd: &str) -> bool {
};
output.success()
}
#[cfg(feature = "proto_gen")]
fn gen_protos() -> Vec<ProtobufCrate> {
let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
let crate_context = ProtoGenerator::gen(&root);
let proto_crates = crate_context
.iter()
.map(|info| info.protobuf_crate.clone())
.collect::<Vec<_>>();
crate_context
.into_iter()
.map(|info| info.files)
.flatten()
.for_each(|file| {
println!("cargo:rerun-if-changed={}", file.file_path);
});
proto_crates
}

2
shared-lib/lib-infra/src/proto_gen/ast.rs
View File

@ -1,6 +1,6 @@
use crate::proto_gen::proto_info::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
use crate::proto_gen::template::{EnumTemplate, StructTemplate};
use crate::proto_gen::util::*;
use crate::proto_gen::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
use fancy_regex::Regex;
use flowy_ast::*;
use lazy_static::lazy_static;

5
shared-lib/lib-infra/src/proto_gen/mod.rs
View File

@ -1,8 +1,9 @@
mod ast;
mod flowy_toml;
mod proto_gen;
pub mod proto_info;
mod proto_info;
mod template;
pub mod util;
pub(crate) use proto_gen::*;
pub use proto_gen::*;
pub use proto_info::*;

41
shared-lib/lib-infra/src/proto_gen/proto_gen.rs
View File

@ -2,6 +2,8 @@ use crate::proto_gen::ast::parse_crate_protobuf;
use crate::proto_gen::proto_info::ProtobufCrateContext;
use crate::proto_gen::template::write_derive_meta;
use crate::proto_gen::util::*;
use crate::proto_gen::ProtoFile;
use std::fs::File;
use std::{fs::OpenOptions, io::Write};
pub(crate) struct ProtoGenerator();
@ -16,6 +18,25 @@ impl ProtoGenerator {
crate_info.create_crate_mod_file();
}
let cache = ProtoCache::from_crate_contexts(&crate_contexts);
let cache_str = serde_json::to_string(&cache).unwrap();
let protobuf_dart = format!("{}/.proto_cache", root);
match std::fs::OpenOptions::new()
.create(true)
.write(true)
.append(false)
.truncate(true)
.open(&protobuf_dart)
{
Ok(ref mut file) => {
file.write_all(cache_str.as_bytes()).unwrap();
File::flush(file).unwrap();
}
Err(err) => {
panic!("Failed to open file: {}", err);
}
}
crate_contexts
}
}
@ -61,3 +82,23 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) {
}
}
}
#[derive(serde::Serialize)]
pub struct ProtoCache {
pub structs: Vec<String>,
pub enums: Vec<String>,
}
impl ProtoCache {
fn from_crate_contexts(crate_contexts: &[ProtobufCrateContext]) -> Self {
let proto_files = crate_contexts
.iter()
.map(|ref crate_info| &crate_info.files)
.flatten()
.collect::<Vec<&ProtoFile>>();
let structs: Vec<String> = proto_files.iter().map(|info| info.structs.clone()).flatten().collect();
let enums: Vec<String> = proto_files.iter().map(|info| info.enums.clone()).flatten().collect();
Self { structs, enums }
}
}