refactor: add code_gen crate

This commit is contained in:
appflowy 2022-02-15 21:27:00 +08:00
parent 427d2c2e25
commit 0933935071
36 changed files with 343 additions and 114 deletions

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files("flowy-error", "./src/protobuf/proto");
code_gen::protobuf_file::gen("flowy-error", "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,7 +1,7 @@
use dashmap::{DashMap, DashSet};
use flowy_ast::{Ctxt, TyInfo};
use lazy_static::lazy_static;
use lib_infra::proto_gen::ProtoCache;
use lib_infra::code_gen::protobuf_file::ProtoCache;
use std::fs::File;
use std::io::Read;
use std::sync::atomic::{AtomicBool, Ordering};

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}

View File

@ -48,4 +48,5 @@ proto_gen = [
"toml"
]
pb_gen = ["cmd_lib", "protoc-rust", "walkdir", "protoc-bin-vendored",]
dart_event = ["walkdir", "flowy-ast", "tera", "syn"]
dart = ["proto_gen"]

View File

@ -0,0 +1,120 @@
use super::event_template::*;
use crate::code_gen::flowy_toml::{parse_crate_config_from, CrateConfig};
use crate::code_gen::util::{is_crate_dir, is_hidden, read_file, save_content_to_file_with_diff_prompt};
use flowy_ast::{event_ast::*, *};
use syn::Item;
use walkdir::WalkDir;
pub struct DartEventCodeGen();
impl DartEventCodeGen {
pub fn gen(crate_name: &str, crate_path: &str) {
let event_crates = parse_dart_event_files(vec![crate_path.to_owned()]);
let event_ast = event_crates.iter().map(parse_event_crate).flatten().collect::<Vec<_>>();
let event_render_ctx = ast_to_event_render_ctx(event_ast.as_ref());
let mut render_result = String::new();
for (index, render_ctx) in event_render_ctx.into_iter().enumerate() {
let mut event_template = EventTemplate::new();
if let Some(content) = event_template.render(render_ctx, index) {
render_result.push_str(content.as_ref())
}
}
save_content_to_file_with_diff_prompt(render_result.as_ref(), ".");
}
}
pub struct DartEventCrate {
crate_path: String,
#[allow(dead_code)]
crate_name: String,
event_files: Vec<String>,
}
impl DartEventCrate {
pub fn from_config(config: &CrateConfig) -> Self {
DartEventCrate {
crate_path: config.crate_path.clone(),
crate_name: config.folder_name.clone(),
event_files: config.flowy_config.event_files.clone(),
}
}
}
pub fn parse_dart_event_files(crate_paths: Vec<String>) -> Vec<DartEventCrate> {
let mut dart_event_crates: Vec<DartEventCrate> = vec![];
crate_paths.iter().for_each(|root| {
let crates = WalkDir::new(root)
.into_iter()
.filter_entry(|e| !is_hidden(e))
.filter_map(|e| e.ok())
.filter(is_crate_dir)
.flat_map(|e| parse_crate_config_from(&e))
.map(|crate_config| DartEventCrate::from_config(&crate_config))
.collect::<Vec<DartEventCrate>>();
dart_event_crates.extend(crates);
});
dart_event_crates
}
pub fn parse_event_crate(event_crate: &DartEventCrate) -> Vec<EventASTContext> {
event_crate
.event_files
.iter()
.map(|event_file| {
let file_path = format!("{}/{}", event_crate.crate_path, event_file);
let file_content = read_file(file_path.as_ref()).unwrap();
let ast = syn::parse_file(file_content.as_ref()).expect("Unable to parse file");
ast.items
.iter()
.map(|item| match item {
Item::Enum(item_enum) => {
let ctxt = Ctxt::new();
let attrs =
flowy_ast::enum_from_ast(&ctxt, &item_enum.ident, &item_enum.variants, &item_enum.attrs);
ctxt.check().unwrap();
attrs
.iter()
.filter(|attr| !attr.attrs.event_attrs.ignore)
.enumerate()
.map(|(_index, attr)| EventASTContext::from(&attr.attrs))
.collect::<Vec<_>>()
}
_ => vec![],
})
.flatten()
.collect::<Vec<_>>()
})
.flatten()
.collect::<Vec<EventASTContext>>()
}
pub fn ast_to_event_render_ctx(ast: &[EventASTContext]) -> Vec<EventRenderContext> {
ast.iter()
.map(|event_ast| {
let input_deserializer = event_ast
.event_input
.as_ref()
.map(|event_input| event_input.get_ident().unwrap().to_string());
let output_deserializer = event_ast
.event_output
.as_ref()
.map(|event_output| event_output.get_ident().unwrap().to_string());
// eprintln!(
// "😁 {:?} / {:?}",
// event_ast.event_input, event_ast.event_output
// );
EventRenderContext {
input_deserializer,
output_deserializer,
error_deserializer: event_ast.event_error.clone(),
event: event_ast.event.to_string(),
event_ty: event_ast.event_ty.to_string(),
}
})
.collect::<Vec<EventRenderContext>>()
}

View File

@ -0,0 +1,69 @@
use crate::code_gen::util::get_tera;
use tera::Context;
pub struct EventTemplate {
tera_context: Context,
}
pub const DART_IMPORTED: &str = r#"
/// Auto gen code from rust ast, do not edit
part of 'dispatch.dart';
"#;
pub struct EventRenderContext {
pub input_deserializer: Option<String>,
pub output_deserializer: Option<String>,
pub error_deserializer: String,
pub event: String,
pub event_ty: String,
}
#[allow(dead_code)]
impl EventTemplate {
pub fn new() -> Self {
EventTemplate {
tera_context: Context::new(),
}
}
pub fn render(&mut self, ctx: EventRenderContext, index: usize) -> Option<String> {
if index == 0 {
self.tera_context.insert("imported_dart_files", DART_IMPORTED)
}
self.tera_context.insert("index", &index);
let dart_class_name = format!("{}{}", ctx.event_ty, ctx.event);
let event = format!("{}.{}", ctx.event_ty, ctx.event);
self.tera_context.insert("event_class", &dart_class_name);
self.tera_context.insert("event", &event);
self.tera_context.insert("has_input", &ctx.input_deserializer.is_some());
match ctx.input_deserializer {
None => self.tera_context.insert("input_deserializer", "Unit"),
Some(ref input) => self.tera_context.insert("input_deserializer", input),
}
// eprintln!(
// "😁 {:?} / {:?}",
// &ctx.input_deserializer, &ctx.output_deserializer
// );
let has_output = ctx.output_deserializer.is_some();
self.tera_context.insert("has_output", &has_output);
match ctx.output_deserializer {
None => self.tera_context.insert("output_deserializer", "Unit"),
Some(ref output) => self.tera_context.insert("output_deserializer", output),
}
self.tera_context.insert("error_deserializer", &ctx.error_deserializer);
let tera = get_tera("dart_event");
match tera.render("event_template.tera", &self.tera_context) {
Ok(r) => Some(r),
Err(e) => {
log::error!("{:?}", e);
None
}
}
}
}

View File

@ -0,0 +1,49 @@
{%- if index == 0 %}
{{ imported_dart_files }}
{%- endif -%}
class {{ event_class }} {
{%- if has_input %}
{{ input_deserializer }} request;
{{ event_class }}(this.request);
{%- else %}
{{ event_class }}();
{%- endif %}
Future<Either<{{ output_deserializer }}, {{ error_deserializer }}>> send() {
{%- if has_input %}
final request = FFIRequest.create()
..event = {{ event }}.toString()
..payload = requestToBytes(this.request);
return Dispatch.asyncRequest(request)
.then((bytesResult) => bytesResult.fold(
{%- if has_output %}
(okBytes) => left({{ output_deserializer }}.fromBuffer(okBytes)),
{%- else %}
(bytes) => left(unit),
{%- endif %}
(errBytes) => right({{ error_deserializer }}.fromBuffer(errBytes)),
));
{%- else %}
final request = FFIRequest.create()
..event = {{ event }}.toString();
{%- if has_input %}
..payload = bytes;
{%- endif %}
return Dispatch.asyncRequest(request).then((bytesResult) => bytesResult.fold(
{%- if has_output %}
(okBytes) => left({{ output_deserializer }}.fromBuffer(okBytes)),
{%- else %}
(bytes) => left(unit),
{%- endif %}
(errBytes) => right({{ error_deserializer }}.fromBuffer(errBytes)),
));
{%- endif %}
}
}

View File

@ -0,0 +1,5 @@
#![allow(clippy::module_inception)]
mod dart_event;
mod event_template;
pub use dart_event::*;

View File

@ -15,9 +15,9 @@ impl FlowyConfig {
}
pub struct CrateConfig {
pub(crate) crate_path: String,
pub(crate) folder_name: String,
pub(crate) flowy_config: FlowyConfig,
pub crate_path: String,
pub folder_name: String,
pub flowy_config: FlowyConfig,
}
impl CrateConfig {

View File

@ -0,0 +1,11 @@
#[cfg(feature = "pb_gen")]
pub mod protobuf_file;
#[cfg(feature = "dart_event")]
pub mod dart_event;
#[cfg(any(feature = "pb_gen", feature = "dart_event"))]
mod flowy_toml;
#[cfg(any(feature = "pb_gen", feature = "dart_event"))]
mod util;

View File

@ -2,9 +2,9 @@
#![allow(dead_code)]
#![allow(unused_imports)]
#![allow(unused_results)]
use crate::proto_gen::template::{EnumTemplate, StructTemplate};
use crate::proto_gen::util::*;
use crate::proto_gen::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
use crate::code_gen::protobuf_file::template::{EnumTemplate, StructTemplate};
use crate::code_gen::protobuf_file::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
use crate::code_gen::util::*;
use fancy_regex::Regex;
use flowy_ast::*;
use lazy_static::lazy_static;
@ -12,8 +12,8 @@ use std::{fs::File, io::Read, path::Path};
use syn::Item;
use walkdir::WalkDir;
pub fn parse_crate_protobuf(roots: Vec<String>) -> Vec<ProtobufCrateContext> {
let crate_infos = parse_crate_info_from_path(roots);
pub fn parse_crate_protobuf(crate_paths: Vec<String>) -> Vec<ProtobufCrateContext> {
let crate_infos = parse_crate_info_from_path(crate_paths);
crate_infos
.into_iter()
.map(|crate_info| {

View File

@ -1,9 +1,15 @@
#![allow(unused_imports)]
#![allow(unused_attributes)]
#![allow(dead_code)]
mod ast;
mod proto_gen;
mod proto_info;
mod template;
pub use proto_gen::*;
pub use proto_info::*;
#[cfg(feature = "proto_gen")]
use crate::proto_gen::*;
use log::info;
use std::fs::File;
use std::io::Write;
@ -11,7 +17,7 @@ use std::path::PathBuf;
use std::process::Command;
use walkdir::WalkDir;
pub fn gen_files(crate_name: &str, proto_file_dir: &str) {
pub fn gen(crate_name: &str, proto_file_dir: &str) {
// 1. generate the proto files to proto_file_dir
#[cfg(feature = "proto_gen")]
let _ = gen_protos(crate_name);

View File

@ -2,17 +2,17 @@
#![allow(dead_code)]
#![allow(unused_imports)]
#![allow(unused_results)]
use crate::proto_gen::ast::parse_crate_protobuf;
use crate::proto_gen::proto_info::ProtobufCrateContext;
use crate::proto_gen::util::*;
use crate::proto_gen::ProtoFile;
use crate::code_gen::protobuf_file::ast::parse_crate_protobuf;
use crate::code_gen::protobuf_file::proto_info::ProtobufCrateContext;
use crate::code_gen::protobuf_file::ProtoFile;
use crate::code_gen::util::*;
use std::fs::File;
use std::path::Path;
use std::{fs::OpenOptions, io::Write};
pub(crate) struct ProtoGenerator();
pub struct ProtoGenerator();
impl ProtoGenerator {
pub(crate) fn gen(crate_name: &str, crate_path: &str, cache_path: &str) -> Vec<ProtobufCrateContext> {
pub fn gen(crate_name: &str, crate_path: &str, cache_path: &str) -> Vec<ProtobufCrateContext> {
let crate_contexts = parse_crate_protobuf(vec![crate_path.to_owned()]);
write_proto_files(&crate_contexts);
write_rust_crate_mod_file(&crate_contexts);

View File

@ -1,6 +1,6 @@
#![allow(dead_code)]
use crate::proto_gen::flowy_toml::{parse_crate_config_from, CrateConfig};
use crate::proto_gen::util::*;
use crate::code_gen::flowy_toml::{parse_crate_config_from, CrateConfig};
use crate::code_gen::util::*;
use std::fs::OpenOptions;
use std::io::Write;
use walkdir::WalkDir;

View File

@ -1,4 +1,4 @@
use crate::proto_gen::template::get_tera;
use crate::code_gen::util::get_tera;
use itertools::Itertools;
use tera::Context;
@ -23,7 +23,7 @@ impl ProtobufDeriveMeta {
self.context.insert("names", &self.structs);
self.context.insert("enums", &self.enums);
let tera = get_tera("derive_meta");
let tera = get_tera("protobuf_file/template/derive_meta");
match tera.render("derive_meta.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

View File

@ -0,0 +1,5 @@
mod derive_meta;
mod proto_file;
pub use derive_meta::*;
pub use proto_file::*;

View File

@ -1,5 +1,5 @@
use crate::proto_gen::ast::FlowyEnum;
use crate::proto_gen::template::get_tera;
use crate::code_gen::protobuf_file::ast::FlowyEnum;
use crate::code_gen::util::get_tera;
use tera::Context;
pub struct EnumTemplate {
@ -26,7 +26,7 @@ impl EnumTemplate {
pub fn render(&mut self) -> Option<String> {
self.context.insert("items", &self.items);
let tera = get_tera("proto_file");
let tera = get_tera("protobuf_file/template/proto_file");
match tera.render("enum.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

View File

@ -1,7 +1,6 @@
use crate::code_gen::util::get_tera;
use flowy_ast::*;
use phf::phf_map;
use crate::proto_gen::template::get_tera;
use tera::Context;
// Protobuf data type : https://developers.google.com/protocol-buffers/docs/proto3
@ -95,7 +94,7 @@ impl StructTemplate {
pub fn render(&mut self) -> Option<String> {
self.context.insert("fields", &self.fields);
let tera = get_tera("proto_file");
let tera = get_tera("protobuf_file/template/proto_file");
match tera.render("struct.tera", &self.context) {
Ok(r) => Some(r),
Err(e) => {

View File

@ -1,11 +1,11 @@
use console::Style;
use similar::{ChangeTag, TextDiff};
use std::{
fs::{File, OpenOptions},
io::{Read, Write},
path::Path,
};
use tera::Tera;
use walkdir::WalkDir;
pub fn read_file(path: &str) -> Option<String> {
@ -100,7 +100,7 @@ pub fn create_dir_if_not_exist(dir: &str) {
}
#[allow(dead_code)]
pub(crate) fn walk_dir<F1, F2>(dir: &str, filter: F2, mut path_and_name: F1)
pub fn walk_dir<F1, F2>(dir: &str, filter: F2, mut path_and_name: F1)
where
F1: FnMut(String, String),
F2: Fn(&walkdir::DirEntry) -> bool,
@ -126,3 +126,29 @@ pub fn suffix_relative_to_path(path: &str, base: &str) -> String {
let path = Path::new(path);
path.strip_prefix(base).unwrap().to_str().unwrap().to_owned()
}
pub fn get_tera(directory: &str) -> Tera {
let mut root = format!("{}/src/code_gen/", env!("CARGO_MANIFEST_DIR"));
root.push_str(directory);
let root_absolute_path = match std::fs::canonicalize(&root) {
Ok(p) => p.as_path().display().to_string(),
Err(e) => {
panic!("❌ Canonicalize file path {} failed {:?}", root, e);
}
};
let mut template_path = format!("{}/**/*.tera", root_absolute_path);
if cfg!(windows) {
// remove "\\?\" prefix on windows
template_path = format!("{}/**/*.tera", &root_absolute_path[4..]);
}
match Tera::new(template_path.as_ref()) {
Ok(t) => t,
Err(e) => {
log::error!("Parsing error(s): {}", e);
::std::process::exit(1);
}
}
}

View File

@ -1,12 +1,7 @@
pub mod code_gen;
pub mod future;
pub mod retry;
#[cfg(feature = "pb_gen")]
pub mod pb;
#[cfg(feature = "proto_gen")]
pub mod proto_gen;
#[allow(dead_code)]
pub fn uuid_string() -> String {
uuid::Uuid::new_v4().to_string()

View File

@ -1,10 +0,0 @@
#![allow(clippy::module_inception)]
mod ast;
mod flowy_toml;
mod proto_gen;
mod proto_info;
mod template;
pub mod util;
pub use proto_gen::*;
pub use proto_info::*;

View File

@ -1,47 +0,0 @@
mod derive_meta;
mod proto_file;
pub use derive_meta::*;
pub use proto_file::*;
use std::fs::File;
use std::io::Read;
use tera::Tera;
pub fn get_tera(directory: &str) -> Tera {
let mut root = format!("{}/src/proto_gen/template/", env!("CARGO_MANIFEST_DIR"));
root.push_str(directory);
let root_absolute_path = match std::fs::canonicalize(&root) {
Ok(p) => p.as_path().display().to_string(),
Err(e) => {
panic!("❌ Canonicalize file path {} failed {:?}", root, e);
}
};
let mut template_path = format!("{}/**/*.tera", root_absolute_path);
if cfg!(windows) {
// remove "\\?\" prefix on windows
template_path = format!("{}/**/*.tera", &root_absolute_path[4..]);
}
match Tera::new(template_path.as_ref()) {
Ok(t) => t,
Err(e) => {
log::error!("Parsing error(s): {}", e);
::std::process::exit(1);
}
}
}
#[allow(dead_code)]
pub fn read_file(path: &str) -> Option<String> {
let mut file = File::open(path).unwrap_or_else(|_| panic!("Unable to open file at {}", path));
let mut content = String::new();
match file.read_to_string(&mut content) {
Ok(_) => Some(content),
Err(e) => {
log::error!("{}, with error: {:?}", path, e);
Some("".to_string())
}
}
}

View File

@ -1,5 +1,5 @@
use lib_infra::pb;
use lib_infra::code_gen;
fn main() {
pb::gen_files(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
code_gen::protobuf_file::gen(env!("CARGO_PKG_NAME"), "./src/protobuf/proto");
}