Mirrored_Repos/AppFlowy
1
0
Fork 0
mirror of https://github.com/AppFlowy-IO/AppFlowy.git synced 2024-08-30 18:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: read cache proto info

Browse Source
This commit is contained in:
appflowy 2022-02-09 18:17:06 +08:00
parent 1c3c519b14
commit 4b63170e56
27 changed files with 133 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -15,4 +15,5 @@
package-lock.json
yarn.lock
node_modules
**/.proto_cache
**/.proto_cache
**/.cache

2
frontend/rust-lib/Cargo.lock generated
View File

@ -906,7 +906,9 @@ dependencies = [
"lib-infra",
"proc-macro2",
"quote",
"serde_json",
"syn",
"walkdir",
]
[[package]]

2
frontend/rust-lib/dart-ffi/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("dart-ffi", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
frontend/rust-lib/dart-notify/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("dart-notify", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
frontend/rust-lib/flowy-folder/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-folder", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
frontend/rust-lib/flowy-net/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-net", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
frontend/rust-lib/flowy-user/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-user", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
shared-lib/Cargo.lock generated
View File

@ -450,9 +450,11 @@ dependencies = [
"log",
"proc-macro2",
"quote",
"serde_json",
"syn",
"tokio",
"trybuild",
"walkdir",
]
[[package]]

2
shared-lib/error-code/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("error-code", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
shared-lib/flowy-collaboration/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-collaboration", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

4
shared-lib/flowy-derive/Cargo.toml
View File

@ -20,7 +20,9 @@ proc-macro2 = "1.0"
flowy-ast = { path = "../flowy-ast" }
lazy_static = {version = "1.4.0"}
dashmap = "4.0"
lib-infra = { path = "../lib-infra"}
lib-infra = { path = "../lib-infra", features = ["proto_gen"]}
serde_json = "1.0"
walkdir = "2.3.1"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }

122
shared-lib/flowy-derive/src/derive_cache/derive_cache.rs
View File

@ -1,122 +0,0 @@
#![cfg_attr(rustfmt, rustfmt::skip)]
use lazy_static::lazy_static;
use dashmap::DashMap;
lazy_static! {
static ref map: DashMap<String,String> = DashMap::new();
}
pub enum TypeCategory {
Array,
Map,
Str,
Protobuf,
Bytes,
Enum,
Opt,
Primitive,
}
// auto generate, do not edit
pub fn category_from_str(type_str: &str) -> TypeCategory {
let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
if !map.contains_key(&root_absolute_path) {
println!("😁{}", root_absolute_path);
map.insert(root_absolute_path, "123".to_string());
}
match type_str {
"Vec" => TypeCategory::Array,
"HashMap" => TypeCategory::Map,
"u8" => TypeCategory::Bytes,
"String" => TypeCategory::Str,
"FFIRequest"
| "FFIResponse"
| "FlowyError"
| "SubscribeObject"
| "NetworkState"
| "SignInRequest"
| "SignInParams"
| "SignInResponse"
| "SignUpRequest"
| "SignUpParams"
| "SignUpResponse"
| "UserToken"
| "UserProfile"
| "UpdateUserRequest"
| "UpdateUserParams"
| "UserPreferences"
| "AppearanceSettings"
| "LocaleSettings"
| "ClientRevisionWSData"
| "ServerRevisionWSData"
| "NewDocumentUser"
| "FolderInfo"
| "Revision"
| "RepeatedRevision"
| "RevId"
| "RevisionRange"
| "CreateDocParams"
| "DocumentInfo"
| "ResetDocumentParams"
| "DocumentDelta"
| "NewDocUser"
| "DocumentId"
| "WSError"
| "WebSocketRawMessage"
| "Workspace"
| "RepeatedWorkspace"
| "CreateWorkspaceRequest"
| "CreateWorkspaceParams"
| "QueryWorkspaceRequest"
| "WorkspaceId"
| "CurrentWorkspaceSetting"
| "UpdateWorkspaceRequest"
| "UpdateWorkspaceParams"
| "ExportRequest"
| "ExportData"
| "App"
| "RepeatedApp"
| "CreateAppRequest"
| "ColorStyle"
| "CreateAppParams"
| "QueryAppRequest"
| "AppId"
| "UpdateAppRequest"
| "UpdateAppParams"
| "Trash"
| "RepeatedTrash"
| "RepeatedTrashId"
| "TrashId"
| "View"
| "RepeatedView"
| "CreateViewRequest"
| "CreateViewParams"
| "QueryViewRequest"
| "ViewId"
| "RepeatedViewId"
| "UpdateViewRequest"
| "UpdateViewParams"
=> TypeCategory::Protobuf,
"FFIStatusCode"
| "FolderEvent"
| "FolderNotification"
| "NetworkEvent"
| "NetworkType"
| "UserEvent"
| "UserNotification"
| "ClientRevisionWSDataType"
| "ServerRevisionWSDataType"
| "RevisionState"
| "RevType"
| "ErrorCode"
| "WSChannel"
| "ExportType"
| "TrashType"
| "ViewType"
=> TypeCategory::Enum,
"Option" => TypeCategory::Opt,
_ => TypeCategory::Primitive,
}
}

4
shared-lib/flowy-derive/src/derive_cache/mod.rs
View File

@ -1,4 +0,0 @@
#![allow(clippy::module_inception)]
mod derive_cache;
pub use derive_cache::*;

1
shared-lib/flowy-derive/src/lib.rs
View File

@ -9,7 +9,6 @@ use syn::{parse_macro_input, DeriveInput};
extern crate quote;
mod dart_event;
mod derive_cache;
mod proto_buf;
// Inspired by https://serde.rs/attributes.html

2
shared-lib/flowy-derive/src/proto_buf/deserialize.rs
View File

@ -1,4 +1,4 @@
use crate::{derive_cache::TypeCategory, proto_buf::util::*};
use crate::proto_buf::util::*;
use flowy_ast::*;
use proc_macro2::{Span, TokenStream};

5
shared-lib/flowy-derive/src/proto_buf/serialize.rs
View File

@ -1,8 +1,5 @@
#![allow(clippy::while_let_on_iterator)]
use crate::{
derive_cache::TypeCategory,
proto_buf::util::{get_member_ident, ident_category},
};
use crate::proto_buf::util::{get_member_ident, ident_category, TypeCategory};
use flowy_ast::*;
use proc_macro2::TokenStream;

91
shared-lib/flowy-derive/src/proto_buf/util.rs
View File

@ -1,8 +1,14 @@
use crate::derive_cache::*;
use dashmap::{DashMap, DashSet};
use flowy_ast::{Ctxt, TyInfo};
use lazy_static::lazy_static;
use lib_infra::proto_gen::ProtoCache;
use std::fs::File;
use std::io::Read;
use std::sync::atomic::{AtomicBool, Ordering};
use walkdir::WalkDir;
pub fn ident_category(ident: &syn::Ident) -> TypeCategory {
let ident_str: &str = &ident.to_string();
let ident_str = ident.to_string();
category_from_str(ident_str)
}
@ -20,3 +26,84 @@ pub fn assert_bracket_ty_is_some(ctxt: &Ctxt, ty_info: &TyInfo) {
ctxt.error_spanned_by(ty_info.ty, "Invalid bracketed type when gen de token steam".to_string());
}
}
lazy_static! {
static ref READ_FLAG: DashSet<String> = DashSet::new();
static ref CACHE_INFO: DashMap<TypeCategory, Vec<String>> = DashMap::new();
static ref IS_LOAD: AtomicBool = AtomicBool::new(false);
}
#[derive(Eq, Hash, PartialEq)]
pub enum TypeCategory {
Array,
Map,
Str,
Protobuf,
Bytes,
Enum,
Opt,
Primitive,
}
// auto generate, do not edit
pub fn category_from_str(type_str: String) -> TypeCategory {
if !IS_LOAD.load(Ordering::SeqCst) {
IS_LOAD.store(true, Ordering::SeqCst);
// Dependents on another crate file is not good, just leave it here.
// Maybe find another way to read the .cache in the future.
let cache_dir = format!("{}/../lib-infra/.cache", env!("CARGO_MANIFEST_DIR"));
for path in WalkDir::new(cache_dir)
.into_iter()
.filter_map(|e| e.ok())
.map(|e| e.path().to_str().unwrap().to_string())
{
match read_file(&path) {
None => {}
Some(s) => {
let cache: ProtoCache = serde_json::from_str(&s).unwrap();
CACHE_INFO
.entry(TypeCategory::Protobuf)
.or_insert(vec![])
.extend(cache.structs);
CACHE_INFO
.entry(TypeCategory::Enum)
.or_insert(vec![])
.extend(cache.enums);
}
}
}
}
if let Some(protobuf) = CACHE_INFO.get(&TypeCategory::Protobuf) {
if protobuf.contains(&type_str) {
return TypeCategory::Protobuf;
}
}
if let Some(protobuf) = CACHE_INFO.get(&TypeCategory::Enum) {
if protobuf.contains(&type_str) {
return TypeCategory::Enum;
}
}
match type_str.as_str() {
"Vec" => TypeCategory::Array,
"HashMap" => TypeCategory::Map,
"u8" => TypeCategory::Bytes,
"String" => TypeCategory::Str,
"Option" => TypeCategory::Opt,
_ => TypeCategory::Primitive,
}
}
fn read_file(path: &str) -> Option<String> {
match File::open(path) {
Ok(mut file) => {
let mut content = String::new();
match file.read_to_string(&mut content) {
Ok(_) => Some(content),
Err(_) => None,
}
}
Err(_) => None,
}
}

2
shared-lib/flowy-folder-data-model/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-folder-data-model", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
shared-lib/flowy-user-data-model/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("flowy-user-data-model", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

2
shared-lib/lib-infra/src/lib.rs
View File

@ -5,7 +5,7 @@ pub mod retry;
pub mod pb;
#[cfg(feature = "proto_gen")]
mod proto_gen;
pub mod proto_gen;
#[allow(dead_code)]
pub fn uuid_string() -> String {

11
shared-lib/lib-infra/src/pb.rs
View File

@ -10,9 +10,9 @@ use std::io::Write;
use std::process::Command;
use walkdir::WalkDir;
pub fn gen_files(name: &str, root: &str) {
pub fn gen_files(crate_name: &str, root: &str) {
#[cfg(feature = "proto_gen")]
let _ = gen_protos();
let _ = gen_protos(crate_name);
let mut paths = vec![];
let mut file_names = vec![];
@ -31,7 +31,7 @@ pub fn gen_files(name: &str, root: &str) {
}
println!("cargo:rerun-if-changed=build.rs");
#[cfg(feature = "dart")]
gen_pb_for_dart(name, root, &paths, &file_names);
gen_pb_for_dart(crate_name, root, &paths, &file_names);
protoc_rust::Codegen::new()
.out_dir("./src/protobuf/model")
@ -120,9 +120,10 @@ fn run_command(cmd: &str) -> bool {
}
#[cfg(feature = "proto_gen")]
fn gen_protos() -> Vec<ProtobufCrate> {
fn gen_protos(crate_name: &str) -> Vec<ProtobufCrate> {
let cache_path = env!("CARGO_MANIFEST_DIR");
let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
let crate_context = ProtoGenerator::gen(&root);
let crate_context = ProtoGenerator::gen(crate_name, &root, cache_path);
let proto_crates = crate_context
.iter()
.map(|info| info.protobuf_crate.clone())

18
shared-lib/lib-infra/src/proto_gen/proto_gen.rs
View File

@ -4,12 +4,13 @@ use crate::proto_gen::template::write_derive_meta;
use crate::proto_gen::util::*;
use crate::proto_gen::ProtoFile;
use std::fs::File;
use std::path::Path;
use std::{fs::OpenOptions, io::Write};
pub(crate) struct ProtoGenerator();
impl ProtoGenerator {
pub(crate) fn gen(root: &str) -> Vec<ProtobufCrateContext> {
let crate_contexts = parse_crate_protobuf(vec![root.to_owned()]);
pub(crate) fn gen(crate_name: &str, crate_path: &str, cache_path: &str) -> Vec<ProtobufCrateContext> {
let crate_contexts = parse_crate_protobuf(vec![crate_path.to_owned()]);
write_proto_files(&crate_contexts);
write_rust_crate_mod_file(&crate_contexts);
for crate_info in &crate_contexts {
@ -20,20 +21,25 @@ impl ProtoGenerator {
let cache = ProtoCache::from_crate_contexts(&crate_contexts);
let cache_str = serde_json::to_string(&cache).unwrap();
let protobuf_dart = format!("{}/.proto_cache", root);
let cache_dir = format!("{}/.cache/{}", cache_path, crate_name);
if !Path::new(&cache_dir).exists() {
std::fs::create_dir_all(&cache_dir).unwrap();
}
let protobuf_cache_path = format!("{}/proto_cache", cache_dir);
match std::fs::OpenOptions::new()
.create(true)
.write(true)
.append(false)
.truncate(true)
.open(&protobuf_dart)
.open(&protobuf_cache_path)
{
Ok(ref mut file) => {
file.write_all(cache_str.as_bytes()).unwrap();
File::flush(file).unwrap();
}
Err(err) => {
panic!("Failed to open file: {}", err);
panic!("Failed to open file: {}", protobuf_cache_path);
}
}
@ -83,7 +89,7 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) {
}
}
#[derive(serde::Serialize)]
#[derive(serde::Serialize, serde::Deserialize)]
pub struct ProtoCache {
pub structs: Vec<String>,
pub enums: Vec<String>,

2
shared-lib/lib-infra/src/proto_gen/template/derive_meta/derive_meta.rs
View File

@ -26,7 +26,7 @@ impl ProtobufDeriveMeta {
self.context.insert("names", &self.structs);
self.context.insert("enums", &self.enums);
let tera = get_tera("proto_gen/template/derive_meta");
let tera = get_tera("derive_meta");
match tera.render("derive_meta.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

2
shared-lib/lib-infra/src/proto_gen/template/mod.rs
View File

@ -8,7 +8,7 @@ use std::io::Read;
use tera::Tera;
pub fn get_tera(directory: &str) -> Tera {
let mut root = format!("{}/../shared-lib/lib-infra/src/", env!("CARGO_MAKE_WORKING_DIRECTORY"));
let mut root = format!("{}/../", file!());
root.push_str(directory);
let root_absolute_path = std::fs::canonicalize(root).unwrap().as_path().display().to_string();

2
shared-lib/lib-infra/src/proto_gen/template/proto_file/enum_template.rs
View File

@ -26,7 +26,7 @@ impl EnumTemplate {
pub fn render(&mut self) -> Option<String> {
self.context.insert("items", &self.items);
let tera = get_tera("proto_gen/template/proto_file");
let tera = get_tera("proto_file");
match tera.render("enum.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

2
shared-lib/lib-infra/src/proto_gen/template/proto_file/struct_template.rs
View File

@ -95,7 +95,7 @@ impl StructTemplate {
pub fn render(&mut self) -> Option<String> {
self.context.insert("fields", &self.fields);
let tera = get_tera("proto_gen/template/proto_file");
let tera = get_tera("proto_file");
match tera.render("struct.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

2
shared-lib/lib-ws/build.rs
View File

@ -1,5 +1,5 @@
use lib_infra::pb;
fn main() {
pb::gen_files("lib-ws", "./src/protobuf/proto");
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}