diff --git a/.gitignore b/.gitignore index 088ba6ba7d..f4244d90fe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk +**/target/ \ No newline at end of file diff --git a/.idea/appflowy_client.iml b/.idea/appflowy_client.iml index 505f9c01d4..b16b5270ed 100644 --- a/.idea/appflowy_client.iml +++ b/.idea/appflowy_client.iml @@ -12,6 +12,8 @@ + + @@ -52,6 +54,7 @@ + diff --git a/Makefile.toml b/Makefile.toml index 4f9fb3d977..2757668fb0 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,5 +1,6 @@ extend = [ { path = "scripts/makefile/desktop.toml" }, + { path = "scripts/makefile/protobuf.toml" }, ] [env] diff --git a/app_flowy/.vscode/tasks.json b/app_flowy/.vscode/tasks.json new file mode 100644 index 0000000000..78b1eae1c7 --- /dev/null +++ b/app_flowy/.vscode/tasks.json @@ -0,0 +1,27 @@ +{ + "version": "2.0.0", + "tasks": [ + { + // https://code.visualstudio.com/docs/editor/tasks + //https://gist.github.com/deadalusai/9e13e36d61ec7fb72148 + + // ${workspaceRoot}: the root folder of the team + // ${file}: the current opened file + // ${fileBasename}: the current opened file's basename + // ${fileDirname}: the current opened file's dirname + // ${fileExtname}: the current opened file's extension + // ${cwd}: the current working directory of the spawned process + + "type": "shell", + "command": "sh ${workspaceFolder}/../scripts/build_sdk.sh", + "group": "build", + "options": { + "cwd": "${workspaceFolder}/../" + }, + "problemMatcher": [ + "$rustc" + ], + "label": "build rust sdk" + } + ] +} \ No newline at end of file diff --git a/rust-lib/.cargo/config.toml b/rust-lib/.cargo/config.toml new file mode 100644 index 0000000000..eb93bf7b47 --- /dev/null +++ b/rust-lib/.cargo/config.toml @@ -0,0 +1,2 @@ +#[build] +#target-dir = "./bin" \ No newline at end of file diff --git a/rust-lib/.gitignore b/rust-lib/.gitignore index 08ffdffa38..0f1114ec62 100644 --- a/rust-lib/.gitignore +++ b/rust-lib/.gitignore @@ -8,4 +8,5 @@ Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk -**/**/*.log* \ No newline at end of file +**/**/*.log* +bin/ \ No newline at end of file diff --git a/rust-lib/Cargo.toml b/rust-lib/Cargo.toml index 1e07ba2a25..9e7665ab37 100644 --- a/rust-lib/Cargo.toml +++ b/rust-lib/Cargo.toml @@ -7,6 +7,7 @@ members = [ "flowy-user", "flowy-ast", "flowy-derive", + "flowy-protobuf", ] [profile.dev] diff --git a/rust-lib/flowy-ast/src/ast.rs b/rust-lib/flowy-ast/src/ast.rs index e1f35c910b..5163c07c7c 100644 --- a/rust-lib/flowy-ast/src/ast.rs +++ b/rust-lib/flowy-ast/src/ast.rs @@ -116,7 +116,7 @@ impl<'a> ASTField<'a> { let mut bracket_inner_ty = None; let mut bracket_ty = None; let mut bracket_category = Some(BracketCategory::Other); - match parse_ty(&field.ty) { + match parse_ty(cx, &field.ty) { Some(inner) => { match inner.primitive_ty { PrimitiveTy::Map(map_info) => { @@ -131,6 +131,9 @@ impl<'a> ASTField<'a> { PrimitiveTy::Opt => { bracket_category = Some(BracketCategory::Opt); }, + PrimitiveTy::Other => { + bracket_category = Some(BracketCategory::Other); + }, } match *inner.bracket_ty_info { diff --git a/rust-lib/flowy-ast/src/ty_ext.rs b/rust-lib/flowy-ast/src/ty_ext.rs index ee258eebcc..2596ff04da 100644 --- a/rust-lib/flowy-ast/src/ty_ext.rs +++ b/rust-lib/flowy-ast/src/ty_ext.rs @@ -1,5 +1,4 @@ - - +use crate::Ctxt; use syn::{self, AngleBracketedGenericArguments, PathSegment}; #[derive(Eq, PartialEq, Debug)] @@ -7,6 +6,7 @@ pub enum PrimitiveTy { Map(MapInfo), Vec, Opt, + Other, } #[derive(Debug)] @@ -39,7 +39,7 @@ impl<'a> TyInfo<'a> { } } -pub fn parse_ty(ty: &syn::Type) -> Option { +pub fn parse_ty<'a>(ctxt: &Ctxt, ty: &'a syn::Type) -> Option> { // Type -> TypePath -> Path -> PathSegment -> PathArguments -> // AngleBracketedGenericArguments -> GenericArgument -> Type. if let syn::Type::Path(ref p) = ty { @@ -52,19 +52,27 @@ pub fn parse_ty(ty: &syn::Type) -> Option { None => return None, }; + let is_option = seg.ident == "Option"; + return if let syn::PathArguments::AngleBracketed(ref bracketed) = seg.arguments { match seg.ident.to_string().as_ref() { - "HashMap" => generate_hashmap_ty_info(ty, seg, bracketed), - "Vec" => generate_vec_ty_info(seg, bracketed), + "HashMap" => generate_hashmap_ty_info(ctxt, ty, seg, bracketed), + "Vec" => generate_vec_ty_info(ctxt, seg, bracketed), + "Option" => generate_option_ty_info(ty, seg), _ => { panic!("Unsupported ty") }, } } else { - assert_eq!(seg.ident.to_string(), "Option".to_string()); - generate_option_ty_info(ty, seg) + return Some(TyInfo { + ident: &seg.ident, + ty, + primitive_ty: PrimitiveTy::Other, + bracket_ty_info: Box::new(None), + }); }; } + ctxt.error_spanned_by(ty, format!("Unsupported inner type, get inner type fail")); None } @@ -83,6 +91,7 @@ fn parse_bracketed(bracketed: &AngleBracketedGenericArguments) -> Vec<&syn::Type } pub fn generate_hashmap_ty_info<'a>( + ctxt: &Ctxt, ty: &'a syn::Type, path_segment: &'a PathSegment, bracketed: &'a AngleBracketedGenericArguments, @@ -92,14 +101,14 @@ pub fn generate_hashmap_ty_info<'a>( return None; } let types = parse_bracketed(bracketed); - let key = parse_ty(types[0]).unwrap().ident.to_string(); - let value = parse_ty(types[1]).unwrap().ident.to_string(); - let bracket_ty_info = Box::new(parse_ty(&types[1])); + let key = parse_ty(ctxt, types[0]).unwrap().ident.to_string(); + let value = parse_ty(ctxt, types[1]).unwrap().ident.to_string(); + let bracket_ty_info = Box::new(parse_ty(ctxt, &types[1])); return Some(TyInfo { ident: &path_segment.ident, ty, primitive_ty: PrimitiveTy::Map(MapInfo::new(key, value)), - bracket_ty_info: bracket_ty_info, + bracket_ty_info, }); } @@ -107,6 +116,7 @@ fn generate_option_ty_info<'a>( ty: &'a syn::Type, path_segment: &'a PathSegment, ) -> Option> { + assert_eq!(path_segment.ident.to_string(), "Option".to_string()); return Some(TyInfo { ident: &path_segment.ident, ty, @@ -116,6 +126,7 @@ fn generate_option_ty_info<'a>( } fn generate_vec_ty_info<'a>( + ctxt: &Ctxt, path_segment: &'a PathSegment, bracketed: &'a AngleBracketedGenericArguments, ) -> Option> { @@ -123,7 +134,7 @@ fn generate_vec_ty_info<'a>( return None; } if let syn::GenericArgument::Type(ref bracketed_type) = bracketed.args.first().unwrap() { - let bracketed_ty_info = Box::new(parse_ty(&bracketed_type)); + let bracketed_ty_info = Box::new(parse_ty(ctxt, &bracketed_type)); return Some(TyInfo { ident: &path_segment.ident, ty: bracketed_type, diff --git a/rust-lib/flowy-derive/.gitignore b/rust-lib/flowy-derive/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/rust-lib/flowy-derive/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/rust-lib/flowy-derive/Cargo.toml b/rust-lib/flowy-derive/Cargo.toml new file mode 100644 index 0000000000..0b7cc365e7 --- /dev/null +++ b/rust-lib/flowy-derive/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "flowy-derive" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +proc-macro = true +name = "flowy_derive" + +[[test]] +name = "tests" +path = "tests/progress.rs" + +[dependencies] +syn = { version = "1.0.60", features = ["extra-traits", "visit"] } +quote = "1.0" +proc-macro2 = "1.0" +flowy-ast = { path = "../flowy-ast" } + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } +trybuild = "1.0.40" +log = "0.4.11" diff --git a/rust-lib/flowy-derive/src/derive_cache/derive_cache.rs b/rust-lib/flowy-derive/src/derive_cache/derive_cache.rs new file mode 100644 index 0000000000..504d7231d6 --- /dev/null +++ b/rust-lib/flowy-derive/src/derive_cache/derive_cache.rs @@ -0,0 +1,24 @@ +pub enum TypeCategory { + Array, + Map, + Str, + Protobuf, + Bytes, + Enum, + Opt, + Primitive, +} +// auto generate, do not edit +pub fn category_from_str(type_str: &str) -> TypeCategory { + match type_str { + "Vec" => TypeCategory::Array, + "HashMap" => TypeCategory::Map, + "u8" => TypeCategory::Bytes, + "String" => TypeCategory::Str, + "App" + => TypeCategory::Protobuf, + + "Option" => TypeCategory::Opt, + _ => TypeCategory::Primitive, + } +} diff --git a/rust-lib/flowy-derive/src/derive_cache/mod.rs b/rust-lib/flowy-derive/src/derive_cache/mod.rs new file mode 100644 index 0000000000..049d77bd33 --- /dev/null +++ b/rust-lib/flowy-derive/src/derive_cache/mod.rs @@ -0,0 +1,3 @@ +mod derive_cache; + +pub use derive_cache::*; diff --git a/rust-lib/flowy-derive/src/lib.rs b/rust-lib/flowy-derive/src/lib.rs new file mode 100644 index 0000000000..870438d6a6 --- /dev/null +++ b/rust-lib/flowy-derive/src/lib.rs @@ -0,0 +1,35 @@ +// https://docs.rs/syn/1.0.48/syn/struct.DeriveInput.html +#![feature(str_split_once)] +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +#[macro_use] +extern crate quote; + +mod derive_cache; +mod proto_buf; + +// Inspired by https://serde.rs/attributes.html +#[proc_macro_derive(ProtoBuf, attributes(pb))] +pub fn derive_proto_buf(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + proto_buf::expand_derive(&input) + .unwrap_or_else(to_compile_errors) + .into() +} + +#[proc_macro_derive(ProtoBuf_Enum, attributes(pb))] +pub fn derive_proto_buf_enum(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + proto_buf::expand_enum_derive(&input) + .unwrap_or_else(to_compile_errors) + .into() +} + +fn to_compile_errors(errors: Vec) -> proc_macro2::TokenStream { + let compile_errors = errors.iter().map(syn::Error::to_compile_error); + quote!(#(#compile_errors)*) +} diff --git a/rust-lib/flowy-derive/src/proto_buf/deserialize.rs b/rust-lib/flowy-derive/src/proto_buf/deserialize.rs new file mode 100644 index 0000000000..cbd43de95d --- /dev/null +++ b/rust-lib/flowy-derive/src/proto_buf/deserialize.rs @@ -0,0 +1,223 @@ +use crate::{derive_cache::TypeCategory, proto_buf::util::*}; +use flowy_ast::*; +use proc_macro2::{Span, TokenStream}; + +pub fn make_de_token_steam(ctxt: &Ctxt, ast: &ASTContainer) -> Option { + let pb_ty = ast.attrs.pb_struct_type()?; + let struct_ident = &ast.ident; + + let build_take_fields = ast + .data + .all_fields() + .filter(|f| !f.attrs.skip_deserializing()) + .flat_map(|field| { + if let Some(func) = field.attrs.deserialize_with() { + let member = &field.member; + Some(quote! { o.#member=#struct_ident::#func(pb); }) + } else if field.attrs.is_one_of() { + token_stream_for_one_of(ctxt, field) + } else { + token_stream_for_field(ctxt, &field.member, &field.ty, false) + } + }); + + let de_token_stream: TokenStream = quote! { + impl std::convert::TryFrom<&Vec> for #struct_ident { + type Error = String; + fn try_from(bytes: &Vec) -> Result { + let result: ::protobuf::ProtobufResult = ::protobuf::Message::parse_from_bytes(bytes); + match result { + Ok(mut pb) => { + #struct_ident::try_from(&mut pb) + } + Err(e) => Err(format!("{:?}", e)), + } + } + } + + impl std::convert::TryFrom<&mut flowy_protobuf::#pb_ty> for #struct_ident { + type Error = String; + fn try_from(pb: &mut flowy_protobuf::#pb_ty) -> Result { + let mut o = Self::default(); + #(#build_take_fields)* + Ok(o) + } + } + } + .into(); + + Some(de_token_stream) +} + +fn token_stream_for_one_of(ctxt: &Ctxt, field: &ASTField) -> Option { + let member = &field.member; + let ident = get_member_ident(ctxt, member)?; + let ty_info = parse_ty(ctxt, &field.ty)?; + let has_func = format_ident!("has_{}", ident.to_string()); + match ident_category(ty_info.ident) { + TypeCategory::Enum => { + let get_func = format_ident!("get_{}", ident.to_string()); + let ty = ty_info.ty; + Some(quote! { + if pb.#has_func() { + let enum_de_from_pb = #ty::try_from(&mut pb.#get_func()).unwrap(); + o.#member = Some(enum_de_from_pb); + } + }) + }, + + TypeCategory::Primitive => { + let get_func = format_ident!("get_{}", ident.to_string()); + Some(quote! { + if pb.#has_func() { + o.#member=Some(pb.#get_func()); + } + }) + }, + _ => { + let take_func = format_ident!("take_{}", ident.to_string()); + let ty = ty_info.ty; + + Some(quote! { + if pb.#has_func() { + let struct_de_from_pb = #ty::try_from(&mut pb.#take_func()).unwrap(); + o.#member=Some(struct_de_from_pb); + } + }) + }, + } +} + +fn token_stream_for_field( + ctxt: &Ctxt, + member: &syn::Member, + ty: &syn::Type, + is_option: bool, +) -> Option { + let ident = get_member_ident(ctxt, member)?; + let ty_info = parse_ty(ctxt, ty)?; + match ident_category(ty_info.ident) { + TypeCategory::Array => { + assert_bracket_ty_is_some(ctxt, &ty_info); + token_stream_for_vec(ctxt, &member, &ty_info.bracket_ty_info.unwrap()) + }, + TypeCategory::Map => { + assert_bracket_ty_is_some(ctxt, &ty_info); + token_stream_for_map(ctxt, &member, &ty_info.bracket_ty_info.unwrap()) + }, + TypeCategory::Protobuf => { + // if the type wrapped by SingularPtrField, should call take first + let take = syn::Ident::new("take", Span::call_site()); + // inner_type_ty would be the type of the field. (e.g value of AnyData) + let ty = ty_info.ty; + Some(quote! { + let some_value = pb.#member.#take(); + if some_value.is_some() { + let struct_de_from_pb = #ty::try_from(&mut some_value.unwrap()).unwrap(); + o.#member = struct_de_from_pb; + } + }) + }, + + TypeCategory::Enum => { + let ty = ty_info.ty; + Some(quote! { + let enum_de_from_pb = #ty::try_from(&mut pb.#member).unwrap(); + o.#member = enum_de_from_pb; + + }) + }, + TypeCategory::Str => { + let take_ident = + syn::Ident::new(&format!("take_{}", ident.to_string()), Span::call_site()); + if is_option { + Some(quote! { + if pb.#member.is_empty() { + o.#member = None; + } else { + o.#member = Some(pb.#take_ident()); + } + }) + } else { + Some(quote! { + o.#member = pb.#take_ident(); + }) + } + }, + TypeCategory::Opt => { + token_stream_for_field(ctxt, member, ty_info.bracket_ty_info.unwrap().ty, true) + }, + TypeCategory::Primitive | TypeCategory::Bytes => { + // eprintln!("πŸ˜„ #{:?}", &field.name().unwrap()); + if is_option { + Some(quote! { o.#member = Some(pb.#member.clone()); }) + } else { + Some(quote! { o.#member = pb.#member.clone(); }) + } + }, + } +} + +fn token_stream_for_vec( + ctxt: &Ctxt, + member: &syn::Member, + bracketed_type: &TyInfo, +) -> Option { + let ident = get_member_ident(ctxt, member)?; + + match ident_category(bracketed_type.ident) { + TypeCategory::Protobuf => { + let ty = bracketed_type.ty; + // Deserialize from pb struct of type vec, should call take_xx(), get the + // repeated_field and then calling the into_iter。 + let take_ident = format_ident!("take_{}", ident.to_string()); + Some(quote! { + o.#member = pb.#take_ident() + .into_iter() + .map(|mut m| #ty::try_from(&mut m).unwrap()) + .collect(); + }) + }, + TypeCategory::Bytes => { + // Vec + Some(quote! { + o.#member = pb.#member.clone(); + }) + }, + _ => { + // String + let take_ident = format_ident!("take_{}", ident.to_string()); + Some(quote! { + o.#member = pb.#take_ident().into_vec(); + }) + }, + } +} + +fn token_stream_for_map( + ctxt: &Ctxt, + member: &syn::Member, + bracketed_type: &TyInfo, +) -> Option { + let ident = get_member_ident(ctxt, member)?; + + let take_ident = format_ident!("take_{}", ident.to_string()); + let ty = bracketed_type.ty; + + match ident_category(bracketed_type.ident) { + TypeCategory::Protobuf => Some(quote! { + let mut m: std::collections::HashMap = std::collections::HashMap::new(); + pb.#take_ident().into_iter().for_each(|(k,mut v)| { + m.insert(k.clone(), #ty::try_from(&mut v).unwrap()); + }); + o.#member = m; + }), + _ => Some(quote! { + let mut m: std::collections::HashMap = std::collections::HashMap::new(); + pb.#take_ident().into_iter().for_each(|(k,mut v)| { + m.insert(k.clone(), v); + }); + o.#member = m; + }), + } +} diff --git a/rust-lib/flowy-derive/src/proto_buf/enum_serde.rs b/rust-lib/flowy-derive/src/proto_buf/enum_serde.rs new file mode 100644 index 0000000000..67013aa5cf --- /dev/null +++ b/rust-lib/flowy-derive/src/proto_buf/enum_serde.rs @@ -0,0 +1,32 @@ +use flowy_ast::*; +use proc_macro2::TokenStream; + +#[allow(dead_code)] +pub fn make_enum_token_stream(_ctxt: &Ctxt, cont: &ASTContainer) -> Option { + let enum_ident = &cont.ident; + let pb_enum = cont.attrs.pb_enum_type()?; + let build_to_pb_enum = cont.data.all_idents().map(|i| { + let token_stream: TokenStream = quote! { + #enum_ident::#i => #pb_enum::#i, + }; + token_stream + }); + + let build_from_pb_enum = cont.data.all_idents().map(|i| { + let token_stream: TokenStream = quote! { + #pb_enum::#i => #enum_ident::#i, + }; + token_stream + }); + + Some(quote! { + impl std::convert::TryFrom<#pb_enum> for #enum_ident { + type Error = String; + fn try_from(pb: #pb_enum) -> Result { + match field_type { + #(#build_from_pb_enum)* + } + } + } + }) +} diff --git a/rust-lib/flowy-derive/src/proto_buf/mod.rs b/rust-lib/flowy-derive/src/proto_buf/mod.rs new file mode 100644 index 0000000000..41ab86a803 --- /dev/null +++ b/rust-lib/flowy-derive/src/proto_buf/mod.rs @@ -0,0 +1,42 @@ +mod deserialize; +mod enum_serde; +mod util; +use crate::proto_buf::{deserialize::make_de_token_steam, enum_serde::make_enum_token_stream}; +use flowy_ast::*; +use proc_macro2::TokenStream; + +pub fn expand_derive(input: &syn::DeriveInput) -> Result> { + let ctxt = Ctxt::new(); + let cont = match ASTContainer::from_ast(&ctxt, input) { + Some(cont) => cont, + None => return Err(ctxt.check().unwrap_err()), + }; + + let mut token_stream: TokenStream = TokenStream::default(); + + let de_token_stream = make_de_token_steam(&ctxt, &cont); + if de_token_stream.is_some() { + token_stream.extend(de_token_stream.unwrap()); + } + + ctxt.check()?; + Ok(token_stream) +} + +pub fn expand_enum_derive(input: &syn::DeriveInput) -> Result> { + let ctxt = Ctxt::new(); + let cont = match ASTContainer::from_ast(&ctxt, input) { + Some(cont) => cont, + None => return Err(ctxt.check().unwrap_err()), + }; + + let mut token_stream: TokenStream = TokenStream::default(); + + let enum_token_stream = make_enum_token_stream(&ctxt, &cont); + if enum_token_stream.is_some() { + token_stream.extend(enum_token_stream.unwrap()); + } + + ctxt.check()?; + Ok(token_stream) +} diff --git a/rust-lib/flowy-derive/src/proto_buf/protobuf_trait.rs b/rust-lib/flowy-derive/src/proto_buf/protobuf_trait.rs deleted file mode 100644 index 00364624b2..0000000000 --- a/rust-lib/flowy-derive/src/proto_buf/protobuf_trait.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub trait SerializeProtoBuf { - type ProtoBufType; - fn to_protobuf(&self) -> Self::ProtoBufType; -} - -pub trait DeserializeProtoBuf { - type ProtoBufType; - type ObjectType; - fn from_protobuf(pb: &mut Self::ProtoBufType) -> Self::ObjectType; -} diff --git a/rust-lib/flowy-derive/src/proto_buf/util.rs b/rust-lib/flowy-derive/src/proto_buf/util.rs index 7ffabf2b2e..5d88a83138 100644 --- a/rust-lib/flowy-derive/src/proto_buf/util.rs +++ b/rust-lib/flowy-derive/src/proto_buf/util.rs @@ -1,17 +1,25 @@ -pub enum TypeCategory { - Array, - Map, - Str, - Protobuf, - Bytes, - Enum, - Opt, - Primitive, -} - -fn category_from_str(type_str: &str) -> TypeCategory { TypeCategory::Protobuf } +use crate::derive_cache::*; +use flowy_ast::{Ctxt, TyInfo}; pub fn ident_category(ident: &syn::Ident) -> TypeCategory { let ident_str: &str = &ident.to_string(); category_from_str(ident_str) } + +pub(crate) fn get_member_ident<'a>(ctxt: &Ctxt, member: &'a syn::Member) -> Option<&'a syn::Ident> { + if let syn::Member::Named(ref ident) = member { + Some(ident) + } else { + ctxt.error_spanned_by(member, format!("Unsupported member, shouldn't be self.0")); + None + } +} + +pub fn assert_bracket_ty_is_some(ctxt: &Ctxt, ty_info: &TyInfo) { + if ty_info.bracket_ty_info.is_none() { + ctxt.error_spanned_by( + ty_info.ty, + format!("Invalid bracketed type when gen de token steam"), + ); + } +} diff --git a/rust-lib/flowy-derive/tests/progress.rs b/rust-lib/flowy-derive/tests/progress.rs new file mode 100644 index 0000000000..3dbb147199 --- /dev/null +++ b/rust-lib/flowy-derive/tests/progress.rs @@ -0,0 +1,5 @@ +#[tokio::test] +async fn tests() { + let t = trybuild::TestCases::new(); + t.pass("tests/protobuf_enum.rs"); +} diff --git a/rust-lib/flowy-protobuf/Cargo.toml b/rust-lib/flowy-protobuf/Cargo.toml new file mode 100644 index 0000000000..2aed5eabd2 --- /dev/null +++ b/rust-lib/flowy-protobuf/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "flowy-protobuf" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +protobuf = {version = "2.20.0"} \ No newline at end of file diff --git a/rust-lib/flowy-protobuf/define/user.proto b/rust-lib/flowy-protobuf/define/user.proto new file mode 100644 index 0000000000..161baa2f05 --- /dev/null +++ b/rust-lib/flowy-protobuf/define/user.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +message App { + string id = 1; + string workspace_id = 2; + string name = 3; +} diff --git a/rust-lib/flowy-protobuf/rustfmt.toml b/rust-lib/flowy-protobuf/rustfmt.toml new file mode 100644 index 0000000000..440b6597b6 --- /dev/null +++ b/rust-lib/flowy-protobuf/rustfmt.toml @@ -0,0 +1,4 @@ +max_width = 80 +ignore = [ + "**/define/*.proto", +] \ No newline at end of file diff --git a/rust-lib/flowy-protobuf/src/lib.rs b/rust-lib/flowy-protobuf/src/lib.rs new file mode 100644 index 0000000000..4a7ebf60c1 --- /dev/null +++ b/rust-lib/flowy-protobuf/src/lib.rs @@ -0,0 +1,3 @@ +mod model; + +pub use model::*; diff --git a/rust-lib/flowy-protobuf/src/model/mod.rs b/rust-lib/flowy-protobuf/src/model/mod.rs new file mode 100644 index 0000000000..887ff36376 --- /dev/null +++ b/rust-lib/flowy-protobuf/src/model/mod.rs @@ -0,0 +1,3 @@ + +mod user; +pub use user::*; diff --git a/rust-lib/flowy-protobuf/src/model/user.rs b/rust-lib/flowy-protobuf/src/model/user.rs new file mode 100644 index 0000000000..e2fda8ffff --- /dev/null +++ b/rust-lib/flowy-protobuf/src/model/user.rs @@ -0,0 +1,295 @@ +// This file is generated by rust-protobuf 2.22.1. Do not edit +// @generated + +// https://github.com/rust-lang/rust-clippy/issues/702 +#![allow(unknown_lints)] +#![allow(clippy::all)] + +#![allow(unused_attributes)] +#![cfg_attr(rustfmt, rustfmt::skip)] + +#![allow(box_pointers)] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(trivial_casts)] +#![allow(unused_imports)] +#![allow(unused_results)] +//! Generated file from `user.proto` + +/// Generated files are compatible only with the same version +/// of protobuf runtime. +// const _PROTOBUF_VERSION_CHECK: () = ::protobuf::VERSION_2_22_1; + +#[derive(PartialEq,Clone,Default)] +pub struct App { + // message fields + pub id: ::std::string::String, + pub workspace_id: ::std::string::String, + pub name: ::std::string::String, + // special fields + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, +} + +impl<'a> ::std::default::Default for &'a App { + fn default() -> &'a App { + ::default_instance() + } +} + +impl App { + pub fn new() -> App { + ::std::default::Default::default() + } + + // string id = 1; + + + pub fn get_id(&self) -> &str { + &self.id + } + pub fn clear_id(&mut self) { + self.id.clear(); + } + + // Param is passed by value, moved + pub fn set_id(&mut self, v: ::std::string::String) { + self.id = v; + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_id(&mut self) -> &mut ::std::string::String { + &mut self.id + } + + // Take field + pub fn take_id(&mut self) -> ::std::string::String { + ::std::mem::replace(&mut self.id, ::std::string::String::new()) + } + + // string workspace_id = 2; + + + pub fn get_workspace_id(&self) -> &str { + &self.workspace_id + } + pub fn clear_workspace_id(&mut self) { + self.workspace_id.clear(); + } + + // Param is passed by value, moved + pub fn set_workspace_id(&mut self, v: ::std::string::String) { + self.workspace_id = v; + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_workspace_id(&mut self) -> &mut ::std::string::String { + &mut self.workspace_id + } + + // Take field + pub fn take_workspace_id(&mut self) -> ::std::string::String { + ::std::mem::replace(&mut self.workspace_id, ::std::string::String::new()) + } + + // string name = 3; + + + pub fn get_name(&self) -> &str { + &self.name + } + pub fn clear_name(&mut self) { + self.name.clear(); + } + + // Param is passed by value, moved + pub fn set_name(&mut self, v: ::std::string::String) { + self.name = v; + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_name(&mut self) -> &mut ::std::string::String { + &mut self.name + } + + // Take field + pub fn take_name(&mut self) -> ::std::string::String { + ::std::mem::replace(&mut self.name, ::std::string::String::new()) + } +} + +impl ::protobuf::Message for App { + fn is_initialized(&self) -> bool { + true + } + + fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::ProtobufResult<()> { + while !is.eof()? { + let (field_number, wire_type) = is.read_tag_unpack()?; + match field_number { + 1 => { + ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.id)?; + }, + 2 => { + ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.workspace_id)?; + }, + 3 => { + ::protobuf::rt::read_singular_proto3_string_into(wire_type, is, &mut self.name)?; + }, + _ => { + ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; + }, + }; + } + ::std::result::Result::Ok(()) + } + + // Compute sizes of nested messages + #[allow(unused_variables)] + fn compute_size(&self) -> u32 { + let mut my_size = 0; + if !self.id.is_empty() { + my_size += ::protobuf::rt::string_size(1, &self.id); + } + if !self.workspace_id.is_empty() { + my_size += ::protobuf::rt::string_size(2, &self.workspace_id); + } + if !self.name.is_empty() { + my_size += ::protobuf::rt::string_size(3, &self.name); + } + my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); + self.cached_size.set(my_size); + my_size + } + + fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::ProtobufResult<()> { + if !self.id.is_empty() { + os.write_string(1, &self.id)?; + } + if !self.workspace_id.is_empty() { + os.write_string(2, &self.workspace_id)?; + } + if !self.name.is_empty() { + os.write_string(3, &self.name)?; + } + os.write_unknown_fields(self.get_unknown_fields())?; + ::std::result::Result::Ok(()) + } + + fn get_cached_size(&self) -> u32 { + self.cached_size.get() + } + + fn get_unknown_fields(&self) -> &::protobuf::UnknownFields { + &self.unknown_fields + } + + fn mut_unknown_fields(&mut self) -> &mut ::protobuf::UnknownFields { + &mut self.unknown_fields + } + + fn as_any(&self) -> &dyn (::std::any::Any) { + self as &dyn (::std::any::Any) + } + fn as_any_mut(&mut self) -> &mut dyn (::std::any::Any) { + self as &mut dyn (::std::any::Any) + } + fn into_any(self: ::std::boxed::Box) -> ::std::boxed::Box { + self + } + + fn descriptor(&self) -> &'static ::protobuf::reflect::MessageDescriptor { + Self::descriptor_static() + } + + fn new() -> App { + App::new() + } + + fn descriptor_static() -> &'static ::protobuf::reflect::MessageDescriptor { + static descriptor: ::protobuf::rt::LazyV2<::protobuf::reflect::MessageDescriptor> = ::protobuf::rt::LazyV2::INIT; + descriptor.get(|| { + let mut fields = ::std::vec::Vec::new(); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>( + "id", + |m: &App| { &m.id }, + |m: &mut App| { &mut m.id }, + )); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>( + "workspace_id", + |m: &App| { &m.workspace_id }, + |m: &mut App| { &mut m.workspace_id }, + )); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeString>( + "name", + |m: &App| { &m.name }, + |m: &mut App| { &mut m.name }, + )); + ::protobuf::reflect::MessageDescriptor::new_pb_name::( + "App", + fields, + file_descriptor_proto() + ) + }) + } + + fn default_instance() -> &'static App { + static instance: ::protobuf::rt::LazyV2 = ::protobuf::rt::LazyV2::INIT; + instance.get(App::new) + } +} + +impl ::protobuf::Clear for App { + fn clear(&mut self) { + self.id.clear(); + self.workspace_id.clear(); + self.name.clear(); + self.unknown_fields.clear(); + } +} + +impl ::std::fmt::Debug for App { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + ::protobuf::text_format::fmt(self, f) + } +} + +impl ::protobuf::reflect::ProtobufValue for App { + fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef { + ::protobuf::reflect::ReflectValueRef::Message(self) + } +} + +static file_descriptor_proto_data: &'static [u8] = b"\ + \n\nuser.proto\"L\n\x03App\x12\x0e\n\x02id\x18\x01\x20\x01(\tR\x02id\x12\ + !\n\x0cworkspace_id\x18\x02\x20\x01(\tR\x0bworkspaceId\x12\x12\n\x04name\ + \x18\x03\x20\x01(\tR\x04nameJ\xcf\x01\n\x06\x12\x04\0\0\x06\x01\n\x08\n\ + \x01\x0c\x12\x03\0\0\x12\n\n\n\x02\x04\0\x12\x04\x02\0\x06\x01\n\n\n\x03\ + \x04\0\x01\x12\x03\x02\x08\x0b\n\x0b\n\x04\x04\0\x02\0\x12\x03\x03\x04\ + \x12\n\x0c\n\x05\x04\0\x02\0\x05\x12\x03\x03\x04\n\n\x0c\n\x05\x04\0\x02\ + \0\x01\x12\x03\x03\x0b\r\n\x0c\n\x05\x04\0\x02\0\x03\x12\x03\x03\x10\x11\ + \n\x0b\n\x04\x04\0\x02\x01\x12\x03\x04\x04\x1c\n\x0c\n\x05\x04\0\x02\x01\ + \x05\x12\x03\x04\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\x12\x03\x04\x0b\x17\ + \n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x04\x1a\x1b\n\x0b\n\x04\x04\0\x02\ + \x02\x12\x03\x05\x04\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x05\x04\n\ + \n\x0c\n\x05\x04\0\x02\x02\x01\x12\x03\x05\x0b\x0f\n\x0c\n\x05\x04\0\x02\ + \x02\x03\x12\x03\x05\x12\x13b\x06proto3\ +"; + +static file_descriptor_proto_lazy: ::protobuf::rt::LazyV2<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::rt::LazyV2::INIT; + +fn parse_descriptor_proto() -> ::protobuf::descriptor::FileDescriptorProto { + ::protobuf::Message::parse_from_bytes(file_descriptor_proto_data).unwrap() +} + +pub fn file_descriptor_proto() -> &'static ::protobuf::descriptor::FileDescriptorProto { + file_descriptor_proto_lazy.get(|| { + parse_descriptor_proto() + }) +} diff --git a/rust-lib/flowy-sys/src/request/request.rs b/rust-lib/flowy-sys/src/request/request.rs index 904525292f..460831ef77 100644 --- a/rust-lib/flowy-sys/src/request/request.rs +++ b/rust-lib/flowy-sys/src/request/request.rs @@ -116,29 +116,6 @@ impl ops::DerefMut for Data { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } -// #[cfg(feature = "use_serde")] -// impl FromRequest for Data -// where -// T: serde::de::DeserializeOwned + 'static, -// { -// type Error = SystemError; -// type Future = Ready>; -// -// #[inline] -// fn from_request(req: &EventRequest, payload: &mut Payload) -> -// Self::Future { match payload { -// Payload::None => ready(Err(unexpected_none_payload(req))), -// Payload::Bytes(bytes) => { -// let s = String::from_utf8_lossy(bytes); -// match serde_json::from_str(s.as_ref()) { -// Ok(data) => ready(Ok(Data(data))), -// Err(e) => ready(Err(InternalError::new(format!("{:?}", -// e)).into())), } -// }, -// } -// } -// } - pub trait FromBytes: Sized { fn parse_from_bytes(bytes: &Vec) -> Result; } diff --git a/rust-lib/flowy-user/Cargo.toml b/rust-lib/flowy-user/Cargo.toml index 8b0e3cd3b1..1ad8814744 100644 --- a/rust-lib/flowy-user/Cargo.toml +++ b/rust-lib/flowy-user/Cargo.toml @@ -9,6 +9,8 @@ edition = "2018" derive_more = {version = "0.99", features = ["display"]} flowy-sys = { path = "../flowy-sys" } flowy-log = { path = "../flowy-log" } +flowy-derive = { path = "../flowy-derive" } +flowy-protobuf = { path = "../flowy-protobuf" } tracing = { version = "0.1", features = ["log"] } bytes = "1.0" serde = { version = "1.0", features = ["derive"] } @@ -16,6 +18,7 @@ validator = "0.12.0" rand = { version = "0.8", features=["std_rng"] } unicode-segmentation = "1.7.1" log = "0.4.14" +protobuf = {version = "2.18.0"} [dev-dependencies] quickcheck = "0.9.2" diff --git a/rust-lib/flowy-user/src/domain/user.rs b/rust-lib/flowy-user/src/domain/user.rs index a93f998aeb..4fd2f62e8f 100644 --- a/rust-lib/flowy-user/src/domain/user.rs +++ b/rust-lib/flowy-user/src/domain/user.rs @@ -1,4 +1,5 @@ use crate::domain::{user_email::UserEmail, user_name::UserName}; +use flowy_derive::ProtoBuf; pub struct User { name: UserName, @@ -8,3 +9,15 @@ pub struct User { impl User { pub fn new(name: UserName, email: UserEmail) -> Self { Self { name, email } } } + +#[derive(ProtoBuf, Default)] +pub struct App { + #[pb(index = 1)] + pub id: String, + + #[pb(index = 2)] + pub workspace_id: String, // equal to #[belongs_to(Workspace, foreign_key = "workspace_id")]. + + #[pb(index = 3)] + pub name: String, +} diff --git a/scripts/flowy-tool/.gitignore b/scripts/flowy-tool/.gitignore new file mode 100644 index 0000000000..088ba6ba7d --- /dev/null +++ b/scripts/flowy-tool/.gitignore @@ -0,0 +1,10 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/scripts/flowy-tool/Cargo.toml b/scripts/flowy-tool/Cargo.toml new file mode 100644 index 0000000000..64999206bd --- /dev/null +++ b/scripts/flowy-tool/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "flowy-tool" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "2.33.3" +walkdir = "2.3.1" +syn = { version = "1.0.60", features = ["extra-traits", "parsing", "derive", "full"]} +tera = { version = "1.5.0" } +log = "0.4.11" +env_logger = "0.8.2" +shell = { git="https://github.com/google/rust-shell.git"} +flowy-ast = { path = "../../rust-lib/flowy-ast" } +console = "0.14.0" +fancy-regex = "0.5.0" +lazy_static = "1.4.0" +phf = { version = "0.8.0", features = ["macros"] } +similar = "1.2.2" +dialoguer = "0.8.0" \ No newline at end of file diff --git a/scripts/flowy-tool/src/main.rs b/scripts/flowy-tool/src/main.rs new file mode 100644 index 0000000000..a3918b9ca7 --- /dev/null +++ b/scripts/flowy-tool/src/main.rs @@ -0,0 +1,68 @@ +mod proto; +mod util; + +use clap::{App, Arg}; + +fn main() { + std::env::set_var("RUST_LOG", "Debug"); + env_logger::init(); + + let matches = app().get_matches(); + + if let Some(ref matches) = matches.subcommand_matches("pb-gen") { + let rust_source = matches.value_of("rust_source").unwrap(); + let build_cache = matches.value_of("build_cache").unwrap(); + let rust_mod_dir = matches.value_of("rust_mod_dir").unwrap(); + let flutter_mod_dir = matches.value_of("flutter_mod_dir").unwrap(); + let proto_file_output = matches.value_of("proto_file_output").unwrap(); + + proto::ProtoGen::new() + .set_rust_source_dir(rust_source) + .set_build_cache_dir(build_cache) + .set_rust_mod_dir(rust_mod_dir) + .set_flutter_mod_dir(flutter_mod_dir) + .set_proto_file_output_dir(proto_file_output) + .gen(); + } +} + +pub fn app<'a, 'b>() -> App<'a, 'b> { + let app = App::new("flowy-tool") + .version("0.1") + .author("nathan") + .about("flowy tool") + .subcommand( + App::new("pb-gen") + .about("Generate proto file from rust code") + .arg( + Arg::with_name("rust_source") + .long("rust_source") + .value_name("DIRECTORY") + .help("The directory to the rust code"), + ) + .arg( + Arg::with_name("build_cache") + .long("build_cache") + .value_name("PATH") + .help("Caching information used by flowy-derive"), + ) + .arg( + Arg::with_name("rust_mod_dir") + .long("rust_mod_dir") + .value_name("DIRECTORY"), + ) + .arg( + Arg::with_name("flutter_mod_dir") + .long("flutter_mod_dir") + .value_name("DIRECTORY"), + ) + .arg( + Arg::with_name("proto_file_output") + .long("proto_file_output") + .value_name("DIRECTORY") + .help("The path is used to save the generated proto file"), + ), + ); + + app +} diff --git a/scripts/flowy-tool/src/proto/ast.rs b/scripts/flowy-tool/src/proto/ast.rs new file mode 100644 index 0000000000..c54e923906 --- /dev/null +++ b/scripts/flowy-tool/src/proto/ast.rs @@ -0,0 +1,190 @@ +use crate::proto::helper::*; +use crate::proto::template::{EnumTemplate, StructTemplate}; +use crate::util::*; +use flowy_ast::*; +use syn::Item; +use walkdir::WalkDir; + +pub fn parse_crate_protobuf(root: &str, proto_output_dir: &str) -> Vec { + log::info!("Generate proto file from {}", root); + let domains_info = get_crate_domain_directory(root); + domains_info + .iter() + .map(|domain| { + let files = parse_files_protobuf(&domain.path, proto_output_dir); + CrateProtoInfo::new(&domain, files) + }) + .collect::>() +} + +fn parse_files_protobuf(root: &str, proto_output_dir: &str) -> Vec { + let mut gen_proto_vec: Vec = vec![]; + // file_stem https://doc.rust-lang.org/std/path/struct.Path.html#method.file_stem + for (path, file_name) in WalkDir::new(root) + .into_iter() + .filter_entry(|e| !is_hidden(e)) + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_dir() == false) + .map(|e| { + let path = e.path().to_str().unwrap().to_string(); + let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string(); + (path, file_name) + }) + { + if file_name == "mod" { + continue; + } + + // https://docs.rs/syn/1.0.54/syn/struct.File.html + let ast = + syn::parse_file(read_file(&path).unwrap().as_ref()).expect("Unable to parse file"); + let structs = get_ast_structs(&ast); + + // println!("😁 {} - {}", path, file_name); + let proto_file_path = format!("{}/{}.proto", proto_output_dir, &file_name); + let mut proto_file_content = parse_or_init_proto_file(proto_file_path.as_ref()); + + structs.iter().for_each(|s| { + let mut struct_template = StructTemplate::new(); + struct_template.set_message_struct_name(&s.name); + + s.fields + .iter() + .filter(|f| f.attrs.pb_index().is_some()) + .for_each(|f| { + struct_template.set_field(&f); + }); + + let s = struct_template.render().unwrap(); + proto_file_content.push_str(s.as_ref()); + proto_file_content.push_str("\n"); + }); + + let enums = get_ast_enums(&ast); + enums.iter().for_each(|e| { + let mut enum_template = EnumTemplate::new(); + enum_template.set_message_enum(&e); + let s = enum_template.render().unwrap(); + proto_file_content.push_str(s.as_ref()); + proto_file_content.push_str("\n"); + }); + + if !enums.is_empty() || !structs.is_empty() { + let info = FileProtoInfo { + file_name: file_name.clone(), + structs: structs.iter().map(|s| s.name.clone()).collect(), + enums: enums.iter().map(|e| e.name.clone()).collect(), + generated_content: proto_file_content.clone(), + }; + gen_proto_vec.push(info); + } + } + + gen_proto_vec +} + +pub fn parse_or_init_proto_file(path: &str) -> String { + let mut proto_file_content = String::new(); + let imported_content = find_proto_file_import(path); + proto_file_content.push_str(imported_content.as_ref()); + proto_file_content.push_str("\n"); + proto_file_content +} + +pub fn get_ast_structs(ast: &syn::File) -> Vec { + // let mut content = format!("{:#?}", &ast); + // let mut file = File::create("./foo.txt").unwrap(); + // file.write_all(content.as_bytes()).unwrap(); + let ctxt = Ctxt::new(); + let mut proto_structs: Vec = vec![]; + ast.items.iter().for_each(|item| match item { + Item::Struct(item_struct) => { + let (_, fields) = struct_from_ast(&ctxt, &item_struct.fields); + + if fields + .iter() + .filter(|f| f.attrs.pb_index().is_some()) + .count() + > 0 + { + proto_structs.push(Struct { + name: item_struct.ident.to_string(), + fields, + }); + } + } + _ => {} + }); + ctxt.check().unwrap(); + proto_structs +} + +pub fn get_ast_enums(ast: &syn::File) -> Vec { + let mut flowy_enums: Vec = vec![]; + let ctxt = Ctxt::new(); + ast.items.iter().for_each(|item| { + // https://docs.rs/syn/1.0.54/syn/enum.Item.html + match item { + Item::Enum(item_enum) => { + flowy_enums.push(FlowyEnum { + name: item_enum.ident.to_string(), + attrs: flowy_ast::enum_from_ast(&ctxt, &item_enum.variants), + }); + } + _ => {} + } + }); + ctxt.check().unwrap(); + flowy_enums +} + +pub struct FlowyEnum<'a> { + pub name: String, + pub attrs: Vec>, +} + +pub struct Struct<'a> { + pub name: String, + pub fields: Vec>, +} + +use fancy_regex::Regex; +use lazy_static::lazy_static; +use std::{fs::File, io::Read, path::Path}; + +lazy_static! { + static ref SYNTAX_REGEX: Regex = Regex::new("syntax.*;").unwrap(); + static ref IMPORT_REGEX: Regex = Regex::new("(import\\s).*;").unwrap(); +} + +fn find_proto_file_import(path: &str) -> String { + let mut result = String::new(); + if !Path::new(path).exists() { + // log::error!("{} not exist", path); + result = String::from("syntax = \"proto3\";"); + return result; + } + + let mut file = File::open(path).unwrap(); + let mut content = String::new(); + file.read_to_string(&mut content).unwrap(); + + content.lines().for_each(|line| { + ////Result>> + if let Ok(some_line) = SYNTAX_REGEX.find(line) { + if let Some(m) = some_line { + result.push_str(m.as_str()); + result.push_str("\n"); + } + } + + if let Ok(some_line) = IMPORT_REGEX.find(line) { + if let Some(m) = some_line { + result.push_str(m.as_str()); + result.push_str("\n"); + } + } + }); + + result +} diff --git a/scripts/flowy-tool/src/proto/helper.rs b/scripts/flowy-tool/src/proto/helper.rs new file mode 100644 index 0000000000..78f9a39558 --- /dev/null +++ b/scripts/flowy-tool/src/proto/helper.rs @@ -0,0 +1,71 @@ +use walkdir::WalkDir; + +pub struct CrateInfo { + pub name: String, + pub path: String, +} + +pub struct CrateProtoInfo { + pub files: Vec, + pub name: String, + pub path: String, +} + +impl CrateProtoInfo { + pub fn new(info: &CrateInfo, files: Vec) -> Self { + Self { + files, + name: info.name.to_owned(), + path: info.path.to_owned(), + } + } +} + +#[derive(Debug)] +pub struct FileProtoInfo { + pub file_name: String, + pub structs: Vec, + pub enums: Vec, + pub generated_content: String, +} + +pub fn get_crate_domain_directory(root: &str) -> Vec { + WalkDir::new(root) + .into_iter() + .filter_entry(|e| !is_hidden(e)) + .filter_map(|e| e.ok()) + .filter(|e| is_domain_dir(e)) + .map(|e| CrateInfo { + //TODO: get the crate name from toml file + name: e + .path() + .parent() + .unwrap() + .parent() + .unwrap() + .file_stem() + .unwrap() + .to_str() + .unwrap() + .to_string(), + path: e.path().to_str().unwrap().to_string(), + }) + .collect::>() +} + +pub fn is_domain_dir(e: &walkdir::DirEntry) -> bool { + let domain = e.path().file_stem().unwrap().to_str().unwrap().to_string(); + if e.file_type().is_dir() && domain == "domain".to_string() { + true + } else { + false + } +} + +pub fn is_hidden(entry: &walkdir::DirEntry) -> bool { + entry + .file_name() + .to_str() + .map(|s| s.starts_with(".")) + .unwrap_or(false) +} diff --git a/scripts/flowy-tool/src/proto/mod.rs b/scripts/flowy-tool/src/proto/mod.rs new file mode 100644 index 0000000000..ed94e08a24 --- /dev/null +++ b/scripts/flowy-tool/src/proto/mod.rs @@ -0,0 +1,6 @@ +mod ast; +mod helper; +mod proto_gen; +mod template; + +pub use proto_gen::*; diff --git a/scripts/flowy-tool/src/proto/proto_gen.rs b/scripts/flowy-tool/src/proto/proto_gen.rs new file mode 100644 index 0000000000..be69de7f4f --- /dev/null +++ b/scripts/flowy-tool/src/proto/proto_gen.rs @@ -0,0 +1,176 @@ +use crate::proto::ast::*; +use crate::proto::helper::*; +use crate::{proto::template::*, util::*}; +use flowy_ast::*; +use std::{fs::OpenOptions, io::Write}; +use syn::Item; +use walkdir::WalkDir; + +pub struct ProtoGen { + rust_source_dir: Option, + proto_file_output_dir: Option, + rust_mod_dir: Option, + flutter_mod_dir: Option, + build_cache_dir: Option, +} + +impl ProtoGen { + pub fn new() -> Self { + ProtoGen { + rust_source_dir: None, + proto_file_output_dir: None, + rust_mod_dir: None, + flutter_mod_dir: None, + build_cache_dir: None, + } + } + + pub fn set_rust_source_dir(mut self, dir: &str) -> Self { + self.rust_source_dir = Some(dir.to_string()); + self + } + + pub fn set_proto_file_output_dir(mut self, dir: &str) -> Self { + self.proto_file_output_dir = Some(dir.to_string()); + self + } + + pub fn set_rust_mod_dir(mut self, dir: &str) -> Self { + self.rust_mod_dir = Some(dir.to_string()); + self + } + + pub fn set_flutter_mod_dir(mut self, dir: &str) -> Self { + self.flutter_mod_dir = Some(dir.to_string()); + self + } + + pub fn set_build_cache_dir(mut self, build_cache_dir: &str) -> Self { + self.build_cache_dir = Some(build_cache_dir.to_string()); + self + } + + pub fn gen(&self) { + let infos = parse_crate_protobuf( + self.rust_source_dir.as_ref().unwrap().as_ref(), + self.proto_file_output_dir.as_ref().unwrap().as_ref(), + ); + self.write_proto_files(&infos); + self.gen_derive(&infos); + self.update_rust_flowy_protobuf_mod_file(&infos); + } + + fn gen_derive(&self, crate_infos: &Vec) { + let file_proto_infos = crate_infos + .iter() + .map(|ref crate_info| &crate_info.files) + .flatten() + .collect::>(); + + let structs: Vec = file_proto_infos + .iter() + .map(|info| info.structs.clone()) + .flatten() + .collect(); + let enums: Vec = file_proto_infos + .iter() + .map(|info| info.enums.clone()) + .flatten() + .collect(); + let derive_file = self.build_cache_dir.as_ref().unwrap().clone(); + + let mut derive_template = ProtobufDeriveCache::new(structs, enums); + let new_content = derive_template.render().unwrap(); + let old_content = read_file(derive_file.as_ref()).unwrap(); + if new_content.clone() == old_content { + return; + } + // println!("{}", diff_lines(&old_content, &new_content)); + match OpenOptions::new() + .create(true) + .write(true) + .append(false) + .truncate(true) + .open(&derive_file) + { + Ok(ref mut file) => { + file.write_all(new_content.as_bytes()).unwrap(); + } + Err(err) => { + panic!("Failed to open log file: {}", err); + } + } + } + + fn write_proto_files(&self, crate_infos: &Vec) { + for crate_info in crate_infos { + crate_info.files.iter().for_each(|info| { + // let dir = format!( + // "{}/{}", + // self.proto_file_output_dir.as_ref().unwrap(), + // &crate_info.name, + // ); + let dir = format!("{}", self.proto_file_output_dir.as_ref().unwrap(),); + + if !std::path::Path::new(&dir).exists() { + std::fs::create_dir_all(&dir).unwrap(); + } + + let proto_file_path = format!("{}/{}.proto", dir, &info.file_name); + let new_content = info.generated_content.clone(); + save_content_to_file_with_diff_prompt( + &new_content, + proto_file_path.as_ref(), + false, + ); + }); + } + } + + fn update_rust_flowy_protobuf_mod_file(&self, crate_infos: &Vec) { + for crate_info in crate_infos { + // let dir = format!( + // "{}/{}-pb", + // self.rust_mod_dir.as_ref().unwrap(), + // &crate_info.name, + // ); + + let dir = format!("{}/model", self.rust_mod_dir.as_ref().unwrap(),); + if !std::path::Path::new(&dir).exists() { + std::fs::create_dir_all(&dir).unwrap(); + } + let mod_path = format!("{}/mod.rs", dir); + + match OpenOptions::new() + .create(false) + .write(true) + .append(false) + .truncate(true) + .open(&mod_path) + { + Ok(ref mut file) => { + let mut mod_file_content = String::new(); + for (_, file_name) in + WalkDir::new(self.proto_file_output_dir.as_ref().unwrap().clone()) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_dir() == false) + .map(|e| { + ( + e.path().to_str().unwrap().to_string(), + e.path().file_stem().unwrap().to_str().unwrap().to_string(), + ) + }) + { + let c = format!("\nmod {}; \npub use {}::*; \n", &file_name, &file_name); + mod_file_content.push_str(c.as_ref()); + } + file.write_all(mod_file_content.as_bytes()).unwrap(); + } + Err(err) => { + panic!("Failed to open file: {}", err); + } + } + } + } +} diff --git a/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.rs b/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.rs new file mode 100644 index 0000000000..dfcf8ae2d6 --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.rs @@ -0,0 +1,33 @@ +use crate::util::get_tera; +use tera::{Context, Tera}; + +pub struct ProtobufDeriveCache { + context: Context, + structs: Vec, + enums: Vec, +} + +#[allow(dead_code)] +impl ProtobufDeriveCache { + pub fn new(structs: Vec, enums: Vec) -> Self { + return ProtobufDeriveCache { + context: Context::new(), + structs, + enums, + }; + } + + pub fn render(&mut self) -> Option { + self.context.insert("names", &self.structs); + self.context.insert("enums", &self.enums); + + let tera = get_tera("build_cache"); + match tera.render("derive_cache.tera", &self.context) { + Ok(r) => Some(r), + Err(e) => { + log::error!("{:?}", e); + None + } + } + } +} diff --git a/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.tera b/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.tera new file mode 100644 index 0000000000..647811c28e --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/build_cache/derive_cache.tera @@ -0,0 +1,44 @@ +pub enum TypeCategory { + Array, + Map, + Str, + Protobuf, + Bytes, + Enum, + Opt, + Primitive, +} +// auto generate, do not edit +pub fn category_from_str(type_str: &str) -> TypeCategory { + match type_str { + "Vec" => TypeCategory::Array, + "HashMap" => TypeCategory::Map, + "u8" => TypeCategory::Bytes, + "String" => TypeCategory::Str, +{%- for name in names -%} + {%- if loop.first %} + "{{ name }}" + {%- else %} + | "{{ name }}" + {%- endif -%} + {%- if loop.last %} + => TypeCategory::Protobuf, + {%- endif %} + +{%- endfor %} + +{%- for enum in enums -%} + {%- if loop.first %} + "{{ enum }}" + {%- else %} + | "{{ enum }}" + {%- endif -%} + {%- if loop.last %} + => TypeCategory::Enum, + {%- endif %} +{%- endfor %} + + "Option" => TypeCategory::Opt, + _ => TypeCategory::Primitive, + } +} diff --git a/scripts/flowy-tool/src/proto/template/build_cache/mod.rs b/scripts/flowy-tool/src/proto/template/build_cache/mod.rs new file mode 100644 index 0000000000..049d77bd33 --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/build_cache/mod.rs @@ -0,0 +1,3 @@ +mod derive_cache; + +pub use derive_cache::*; diff --git a/scripts/flowy-tool/src/proto/template/mod.rs b/scripts/flowy-tool/src/proto/template/mod.rs new file mode 100644 index 0000000000..ae487e485a --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/mod.rs @@ -0,0 +1,5 @@ +mod build_cache; +mod proto_file; + +pub use build_cache::*; +pub use proto_file::*; diff --git a/scripts/flowy-tool/src/proto/template/proto_file/enum.tera b/scripts/flowy-tool/src/proto/template/proto_file/enum.tera new file mode 100644 index 0000000000..fd6878a7ec --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/proto_file/enum.tera @@ -0,0 +1,5 @@ +enum {{ enum_name }} { + {%- for item in items %} + {{ item }} + {%- endfor %} +} \ No newline at end of file diff --git a/scripts/flowy-tool/src/proto/template/proto_file/enum_template.rs b/scripts/flowy-tool/src/proto/template/proto_file/enum_template.rs new file mode 100644 index 0000000000..5950a51367 --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/proto_file/enum_template.rs @@ -0,0 +1,38 @@ +use crate::proto::ast::FlowyEnum; +use crate::util::get_tera; +use tera::{Context, Tera}; + +pub struct EnumTemplate { + context: Context, + items: Vec, +} + +#[allow(dead_code)] +impl EnumTemplate { + pub fn new() -> Self { + return EnumTemplate { + context: Context::new(), + items: vec![], + }; + } + + pub fn set_message_enum(&mut self, flowy_enum: &FlowyEnum) { + self.context.insert("enum_name", &flowy_enum.name); + flowy_enum.attrs.iter().for_each(|item| { + self.items + .push(format!("{} = {};", item.attrs.name, item.attrs.value)) + }) + } + + pub fn render(&mut self) -> Option { + self.context.insert("items", &self.items); + let tera = get_tera("proto_file"); + match tera.render("enum.tera", &self.context) { + Ok(r) => Some(r), + Err(e) => { + log::error!("{:?}", e); + None + } + } + } +} diff --git a/scripts/flowy-tool/src/proto/template/proto_file/mod.rs b/scripts/flowy-tool/src/proto/template/proto_file/mod.rs new file mode 100644 index 0000000000..bcdd287ce5 --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/proto_file/mod.rs @@ -0,0 +1,5 @@ +mod enum_template; +mod struct_template; + +pub use enum_template::*; +pub use struct_template::*; diff --git a/scripts/flowy-tool/src/proto/template/proto_file/struct.tera b/scripts/flowy-tool/src/proto/template/proto_file/struct.tera new file mode 100644 index 0000000000..ddeb49153a --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/proto_file/struct.tera @@ -0,0 +1,5 @@ +message {{ struct_name }} { + {%- for field in fields %} + {{ field }} + {%- endfor %} +} \ No newline at end of file diff --git a/scripts/flowy-tool/src/proto/template/proto_file/struct_template.rs b/scripts/flowy-tool/src/proto/template/proto_file/struct_template.rs new file mode 100644 index 0000000000..5a926b22d7 --- /dev/null +++ b/scripts/flowy-tool/src/proto/template/proto_file/struct_template.rs @@ -0,0 +1,96 @@ +use crate::util::get_tera; +use flowy_ast::*; +use phf::phf_map; +use tera::{Context, Tera}; + +// Protobuf data type : https://developers.google.com/protocol-buffers/docs/proto3 +static RUST_TYPE_MAP: phf::Map<&'static str, &'static str> = phf_map! { + "String" => "string", + "i64" => "int64", + "i32" => "int32", + "u64" => "uint64", + "u32" => "uint32", + "Vec" => "repeated", + "f64" => "double", + "HashMap" => "map", +}; + +pub struct StructTemplate { + context: Context, + fields: Vec, +} + +#[allow(dead_code)] +impl StructTemplate { + pub fn new() -> Self { + return StructTemplate { + context: Context::new(), + fields: vec![], + }; + } + + pub fn set_message_struct_name(&mut self, name: &str) { + self.context.insert("struct_name", name); + } + + pub fn set_field(&mut self, field: &ASTField) { + // {{ field_type }} {{ field_name }} = {{index}}; + let name = field.name().unwrap().to_string(); + let index = field.attrs.pb_index().unwrap(); + + let ty: &str = &field.ty_as_str(); + let mut mapped_ty: &str = ty; + + if RUST_TYPE_MAP.contains_key(ty) { + mapped_ty = RUST_TYPE_MAP[ty]; + } + + match field.bracket_category { + Some(ref category) => match category { + BracketCategory::Opt => self.fields.push(format!( + "oneof one_of_{} {{ {} {} = {}; }};", + name, mapped_ty, name, index + )), + BracketCategory::Map((k, v)) => { + let key: &str = k; + let value: &str = v; + self.fields.push(format!( + // map attrs = 1; + "map<{}, {}> {} = {};", + RUST_TYPE_MAP.get(key).unwrap_or(&key), + RUST_TYPE_MAP.get(value).unwrap_or(&value), + name, + index + )); + } + BracketCategory::Vec => { + let bracket_ty: &str = &field.bracket_ty.as_ref().unwrap().to_string(); + if mapped_ty == "u8" && bracket_ty == "Vec" { + self.fields.push(format!("bytes {} = {};", name, index)) + } else { + self.fields.push(format!( + "{} {} {} = {};", + RUST_TYPE_MAP[bracket_ty], mapped_ty, name, index + )) + } + } + BracketCategory::Other => self + .fields + .push(format!("{} {} = {};", mapped_ty, name, index)), + }, + None => {} + } + } + + pub fn render(&mut self) -> Option { + self.context.insert("fields", &self.fields); + let tera = get_tera("proto_file"); + match tera.render("struct.tera", &self.context) { + Ok(r) => Some(r), + Err(e) => { + log::error!("{:?}", e); + None + } + } + } +} diff --git a/scripts/flowy-tool/src/util/file.rs b/scripts/flowy-tool/src/util/file.rs new file mode 100644 index 0000000000..3dc566a5c7 --- /dev/null +++ b/scripts/flowy-tool/src/util/file.rs @@ -0,0 +1,107 @@ +use console::Style; +use dialoguer::Confirm; +use similar::{ChangeTag, TextDiff}; +use std::{ + fs::{File, OpenOptions}, + io::{Read, Write}, + path::Path, +}; +use tera::Tera; + +pub fn read_file(path: &str) -> Option { + let mut file = File::open(path).expect("Unable to open file"); + let mut content = String::new(); + match file.read_to_string(&mut content) { + Ok(_) => Some(content), + Err(e) => { + log::error!("{}, with error: {:?}", path, e); + Some("".to_string()) + } + } +} + +pub fn save_content_to_file_with_diff_prompt(content: &str, output_file: &str, force_write: bool) { + if Path::new(output_file).exists() { + let old_content = read_file(output_file).unwrap(); + let new_content = content.to_owned(); + let write_to_file = || match OpenOptions::new() + .create(true) + .write(true) + .append(false) + .truncate(true) + .open(output_file) + { + Ok(ref mut file) => { + file.write_all(new_content.as_bytes()).unwrap(); + } + Err(err) => { + panic!("Failed to open log file: {}", err); + } + }; + if new_content != old_content { + print_diff(old_content.clone(), new_content.clone()); + if force_write { + write_to_file() + } else { + if Confirm::new().with_prompt("Override?").interact().unwrap() { + write_to_file() + } else { + log::info!("never mind then :("); + } + } + } + } else { + match OpenOptions::new() + .create(true) + .write(true) + .open(output_file) + { + Ok(ref mut file) => file.write_all(content.as_bytes()).unwrap(), + Err(err) => panic!("Open or create file fail: {}", err), + } + } +} + +pub fn print_diff(old_content: String, new_content: String) { + let diff = TextDiff::from_lines(&old_content, &new_content); + for op in diff.ops() { + for change in diff.iter_changes(op) { + let (sign, style) = match change.tag() { + ChangeTag::Delete => ("-", Style::new().red()), + ChangeTag::Insert => ("+", Style::new().green()), + ChangeTag::Equal => (" ", Style::new()), + }; + + match change.tag() { + ChangeTag::Delete => { + print!("{}{}", style.apply_to(sign).bold(), style.apply_to(change)); + } + ChangeTag::Insert => { + print!("{}{}", style.apply_to(sign).bold(), style.apply_to(change)); + } + ChangeTag::Equal => {} + }; + } + println!("---------------------------------------------------"); + } +} + +pub fn get_tera(directory: &str) -> Tera { + let mut root = "./scripts/flowy-tool/src/proto/template/".to_owned(); + root.push_str(directory); + + let root_absolute_path = std::fs::canonicalize(root) + .unwrap() + .as_path() + .display() + .to_string(); + + let template_path = format!("{}/**/*.tera", root_absolute_path); + match Tera::new(template_path.as_ref()) { + Ok(t) => t, + Err(e) => { + log::error!("Parsing error(s): {}", e); + ::std::process::exit(1); + } + } +} diff --git a/scripts/flowy-tool/src/util/mod.rs b/scripts/flowy-tool/src/util/mod.rs new file mode 100644 index 0000000000..529c201fa5 --- /dev/null +++ b/scripts/flowy-tool/src/util/mod.rs @@ -0,0 +1,3 @@ +mod file; + +pub use file::*; diff --git a/scripts/makefile/protobuf.toml b/scripts/makefile/protobuf.toml new file mode 100644 index 0000000000..e1cecbde21 --- /dev/null +++ b/scripts/makefile/protobuf.toml @@ -0,0 +1,29 @@ + +[tasks.pb] +dependencies = ["gen_pb_file", "gen_rust_pb"] + +[tasks.gen_pb_file] +script = [ + """ + pb_gen_bin=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/scripts/flowy-tool/Cargo.toml + rust_source=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/ + + build_cache=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-derive/src/derive_cache/derive_cache.rs + proto_file_output=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-protobuf/define + rust_mod_dir=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-protobuf/src/ + flutter_mod_dir=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/flutter-lib/packages/flowy_protobuf/lib/ + + cargo run --manifest-path ${pb_gen_bin} pb-gen --rust_source=${rust_source} --build_cache=${build_cache} --proto_file_output=${proto_file_output} --rust_mod_dir=${rust_mod_dir} --flutter_mod_dir=${flutter_mod_dir} + """, +] +script_runner = "@shell" + +[tasks.gen_rust_pb] +script = [ + """ + protoc --rust_out=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-protobuf/src/model \ + --proto_path=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-protobuf/define \ + ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/flowy-protobuf/define/*.proto + """, +] +script_runner = "@shell" \ No newline at end of file