refactor: optimize protobuf code gen process

.github/workflows/rust_test.yml

@@ -31,9 +31,6 @@ jobs:
       - name: Install cargo-make
         run: cargo install --force cargo-make
         working-directory: frontend
-      - name: Install protobuf tool
-        run: brew install protobuf
-        working-directory: frontend
       - name: RustLib tests
         run: cargo test --no-default-features
         working-directory: frontend/rust-lib

frontend/Brewfile

@@ -1,3 +1,2 @@
-brew 'protobuf'
 brew 'sqlite3'
 brew 'rustup-init'

frontend/app_flowy/packages/flowy_sdk/lib/protobuf/flowy-folder-data-model/protobuf.dart

@@ -1,7 +1,6 @@
 // Auto-generated, do not edit 
 export './share.pb.dart';
 export './app.pb.dart';
-export './errors.pb.dart';
 export './view.pb.dart';
 export './trash.pb.dart';
 export './workspace.pb.dart';

frontend/rust-lib/Cargo.lock

@@ -1725,6 +1725,7 @@ dependencies = [
  "log",
  "phf 0.8.0",
  "pin-project",
+ "protoc-bin-vendored",
  "protoc-rust",
  "rand 0.8.4",
  "serde",
@@ -2416,6 +2417,56 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "protoc-bin-vendored"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d"
+dependencies = [
+ "protoc-bin-vendored-linux-aarch_64",
+ "protoc-bin-vendored-linux-ppcle_64",
+ "protoc-bin-vendored-linux-x86_32",
+ "protoc-bin-vendored-linux-x86_64",
+ "protoc-bin-vendored-macos-x86_64",
+ "protoc-bin-vendored-win32",
+]
+
+[[package]]
+name = "protoc-bin-vendored-linux-aarch_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435"
+
+[[package]]
+name = "protoc-bin-vendored-linux-ppcle_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_32"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924"
+
+[[package]]
+name = "protoc-bin-vendored-macos-x86_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537"
+
+[[package]]
+name = "protoc-bin-vendored-win32"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804"
+
 [[package]]
 name = "protoc-rust"
 version = "2.25.2"

frontend/scripts/makefile/protobuf.toml

@@ -2,11 +2,8 @@
 [tasks.install_protobuf]
 condition_script = [
     """
-    if [ ! "$(command -v protoc)" ]; then
+    if ! command -v protoc-gen-dart
-        exit 0
+    then
-    fi
-
-    if [ ! "$(command -v protoc-gen-dart)" ]; then
         exit 0
     fi
 
@@ -18,8 +15,6 @@ run_task = { name = ["install_protobuf_compiler"] }
 
 [tasks.install_protobuf_compiler]
 script = """
-brew install protobuf
-
 echo "Install protoc_plugin (Dart)"
 dart pub global activate protoc_plugin
 """
@@ -27,8 +22,6 @@ script_runner = "@shell"
 
 [tasks.install_protobuf_compiler.linux]
 script = """
-sudo apt-get install protobuf-compiler
-
 echo "Install protoc_plugin (Dart)"
 dart pub global activate protoc_plugin
 """

shared-lib/Cargo.lock

@@ -804,6 +804,7 @@ dependencies = [
  "log",
  "phf 0.8.0",
  "pin-project",
+ "protoc-bin-vendored",
  "protoc-rust",
  "rand 0.8.4",
  "serde",
@@ -1280,6 +1281,56 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "protoc-bin-vendored"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "005ca8623e5633e298ad1f917d8be0a44bcf406bf3cde3b80e63003e49a3f27d"
+dependencies = [
+ "protoc-bin-vendored-linux-aarch_64",
+ "protoc-bin-vendored-linux-ppcle_64",
+ "protoc-bin-vendored-linux-x86_32",
+ "protoc-bin-vendored-linux-x86_64",
+ "protoc-bin-vendored-macos-x86_64",
+ "protoc-bin-vendored-win32",
+]
+
+[[package]]
+name = "protoc-bin-vendored-linux-aarch_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb9fc9cce84c8694b6ea01cc6296617b288b703719b725b8c9c65f7c5874435"
+
+[[package]]
+name = "protoc-bin-vendored-linux-ppcle_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d2a07dcf7173a04d49974930ccbfb7fd4d74df30ecfc8762cf2f895a094516"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_32"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d54fef0b04fcacba64d1d80eed74a20356d96847da8497a59b0a0a436c9165b0"
+
+[[package]]
+name = "protoc-bin-vendored-linux-x86_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8782f2ce7d43a9a5c74ea4936f001e9e8442205c244f7a3d4286bd4c37bc924"
+
+[[package]]
+name = "protoc-bin-vendored-macos-x86_64"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5de656c7ee83f08e0ae5b81792ccfdc1d04e7876b1d9a38e6876a9e09e02537"
+
+[[package]]
+name = "protoc-bin-vendored-win32"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9653c3ed92974e34c5a6e0a510864dab979760481714c172e0a34e437cb98804"
+
 [[package]]
 name = "protoc-rust"
 version = "2.25.2"

shared-lib/flowy-folder-data-model/src/protobuf/document

@@ -1,4 +0,0 @@
-#![cfg_attr(rustfmt, rustfmt::skip)]
-// Auto-generated, do not edit
-mod model;
-pub use model::*;

shared-lib/flowy-folder-data-model/src/protobuf/model/errors.rs

@@ -1,144 +0,0 @@
-// This file is generated by rust-protobuf 2.25.2. Do not edit
-// @generated
-
-// https://github.com/rust-lang/rust-clippy/issues/702
-#![allow(unknown_lints)]
-#![allow(clippy::all)]
-
-#![allow(unused_attributes)]
-#![cfg_attr(rustfmt, rustfmt::skip)]
-
-#![allow(box_pointers)]
-#![allow(dead_code)]
-#![allow(missing_docs)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-#![allow(non_upper_case_globals)]
-#![allow(trivial_casts)]
-#![allow(unused_imports)]
-#![allow(unused_results)]
-//! Generated file from `errors.proto`
-
-/// Generated files are compatible only with the same version
-/// of protobuf runtime.
-// const _PROTOBUF_VERSION_CHECK: () = ::protobuf::VERSION_2_25_2;
-
-#[derive(Clone,PartialEq,Eq,Debug,Hash)]
-pub enum ErrorCode {
-    WorkspaceNameInvalid = 0,
-    WorkspaceIdInvalid = 1,
-    AppColorStyleInvalid = 2,
-    WorkspaceDescTooLong = 3,
-    WorkspaceNameTooLong = 4,
-    AppIdInvalid = 10,
-    AppNameInvalid = 11,
-    ViewNameInvalid = 20,
-    ViewThumbnailInvalid = 21,
-    ViewIdInvalid = 22,
-    ViewDescTooLong = 23,
-    ViewDataInvalid = 24,
-    ViewNameTooLong = 25,
-    UserUnauthorized = 100,
-    WsConnectError = 200,
-    InternalError = 1000,
-    RecordNotFound = 1001,
-}
-
-impl ::protobuf::ProtobufEnum for ErrorCode {
-    fn value(&self) -> i32 {
-        *self as i32
-    }
-
-    fn from_i32(value: i32) -> ::std::option::Option<ErrorCode> {
-        match value {
-            0 => ::std::option::Option::Some(ErrorCode::WorkspaceNameInvalid),
-            1 => ::std::option::Option::Some(ErrorCode::WorkspaceIdInvalid),
-            2 => ::std::option::Option::Some(ErrorCode::AppColorStyleInvalid),
-            3 => ::std::option::Option::Some(ErrorCode::WorkspaceDescTooLong),
-            4 => ::std::option::Option::Some(ErrorCode::WorkspaceNameTooLong),
-            10 => ::std::option::Option::Some(ErrorCode::AppIdInvalid),
-            11 => ::std::option::Option::Some(ErrorCode::AppNameInvalid),
-            20 => ::std::option::Option::Some(ErrorCode::ViewNameInvalid),
-            21 => ::std::option::Option::Some(ErrorCode::ViewThumbnailInvalid),
-            22 => ::std::option::Option::Some(ErrorCode::ViewIdInvalid),
-            23 => ::std::option::Option::Some(ErrorCode::ViewDescTooLong),
-            24 => ::std::option::Option::Some(ErrorCode::ViewDataInvalid),
-            25 => ::std::option::Option::Some(ErrorCode::ViewNameTooLong),
-            100 => ::std::option::Option::Some(ErrorCode::UserUnauthorized),
-            200 => ::std::option::Option::Some(ErrorCode::WsConnectError),
-            1000 => ::std::option::Option::Some(ErrorCode::InternalError),
-            1001 => ::std::option::Option::Some(ErrorCode::RecordNotFound),
-            _ => ::std::option::Option::None
-        }
-    }
-
-    fn values() -> &'static [Self] {
-        static values: &'static [ErrorCode] = &[
-            ErrorCode::WorkspaceNameInvalid,
-            ErrorCode::WorkspaceIdInvalid,
-            ErrorCode::AppColorStyleInvalid,
-            ErrorCode::WorkspaceDescTooLong,
-            ErrorCode::WorkspaceNameTooLong,
-            ErrorCode::AppIdInvalid,
-            ErrorCode::AppNameInvalid,
-            ErrorCode::ViewNameInvalid,
-            ErrorCode::ViewThumbnailInvalid,
-            ErrorCode::ViewIdInvalid,
-            ErrorCode::ViewDescTooLong,
-            ErrorCode::ViewDataInvalid,
-            ErrorCode::ViewNameTooLong,
-            ErrorCode::UserUnauthorized,
-            ErrorCode::WsConnectError,
-            ErrorCode::InternalError,
-            ErrorCode::RecordNotFound,
-        ];
-        values
-    }
-
-    fn enum_descriptor_static() -> &'static ::protobuf::reflect::EnumDescriptor {
-        static descriptor: ::protobuf::rt::LazyV2<::protobuf::reflect::EnumDescriptor> = ::protobuf::rt::LazyV2::INIT;
-        descriptor.get(|| {
-            ::protobuf::reflect::EnumDescriptor::new_pb_name::<ErrorCode>("ErrorCode", file_descriptor_proto())
-        })
-    }
-}
-
-impl ::std::marker::Copy for ErrorCode {
-}
-
-impl ::std::default::Default for ErrorCode {
-    fn default() -> Self {
-        ErrorCode::WorkspaceNameInvalid
-    }
-}
-
-impl ::protobuf::reflect::ProtobufValue for ErrorCode {
-    fn as_ref(&self) -> ::protobuf::reflect::ReflectValueRef {
-        ::protobuf::reflect::ReflectValueRef::Enum(::protobuf::ProtobufEnum::descriptor(self))
-    }
-}
-
-static file_descriptor_proto_data: &'static [u8] = b"\
-    \n\x0cerrors.proto*\x86\x03\n\tErrorCode\x12\x18\n\x14WorkspaceNameInval\
-    id\x10\0\x12\x16\n\x12WorkspaceIdInvalid\x10\x01\x12\x18\n\x14AppColorSt\
-    yleInvalid\x10\x02\x12\x18\n\x14WorkspaceDescTooLong\x10\x03\x12\x18\n\
-    \x14WorkspaceNameTooLong\x10\x04\x12\x10\n\x0cAppIdInvalid\x10\n\x12\x12\
-    \n\x0eAppNameInvalid\x10\x0b\x12\x13\n\x0fViewNameInvalid\x10\x14\x12\
-    \x18\n\x14ViewThumbnailInvalid\x10\x15\x12\x11\n\rViewIdInvalid\x10\x16\
-    \x12\x13\n\x0fViewDescTooLong\x10\x17\x12\x13\n\x0fViewDataInvalid\x10\
-    \x18\x12\x13\n\x0fViewNameTooLong\x10\x19\x12\x14\n\x10UserUnauthorized\
-    \x10d\x12\x13\n\x0eWsConnectError\x10\xc8\x01\x12\x12\n\rInternalError\
-    \x10\xe8\x07\x12\x13\n\x0eRecordNotFound\x10\xe9\x07b\x06proto3\
-";
-
-static file_descriptor_proto_lazy: ::protobuf::rt::LazyV2<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::rt::LazyV2::INIT;
-
-fn parse_descriptor_proto() -> ::protobuf::descriptor::FileDescriptorProto {
-    ::protobuf::Message::parse_from_bytes(file_descriptor_proto_data).unwrap()
-}
-
-pub fn file_descriptor_proto() -> &'static ::protobuf::descriptor::FileDescriptorProto {
-    file_descriptor_proto_lazy.get(|| {
-        parse_descriptor_proto()
-    })
-}

shared-lib/flowy-folder-data-model/src/protobuf/model/mod.rs

@@ -7,9 +7,6 @@ pub use share::*;
 mod app;
 pub use app::*;
 
-mod errors;
-pub use errors::*;
-
 mod view;
 pub use view::*;
 

shared-lib/flowy-folder-data-model/src/protobuf/proto/errors.proto

@@ -1,21 +0,0 @@
-syntax = "proto3";
-
-enum ErrorCode {
-    WorkspaceNameInvalid = 0;
-    WorkspaceIdInvalid = 1;
-    AppColorStyleInvalid = 2;
-    WorkspaceDescTooLong = 3;
-    WorkspaceNameTooLong = 4;
-    AppIdInvalid = 10;
-    AppNameInvalid = 11;
-    ViewNameInvalid = 20;
-    ViewThumbnailInvalid = 21;
-    ViewIdInvalid = 22;
-    ViewDescTooLong = 23;
-    ViewDataInvalid = 24;
-    ViewNameTooLong = 25;
-    UserUnauthorized = 100;
-    WsConnectError = 200;
-    InternalError = 1000;
-    RecordNotFound = 1001;
-}

shared-lib/lib-infra/Cargo.toml

@@ -30,7 +30,7 @@ tera = { version = "1.5.0", optional = true}
 itertools = { version = "0.10", optional = true }
 phf = { version = "0.8.0", features = ["macros"], optional = true }
 console = {version = "0.14.0", optional = true}
-
+protoc-bin-vendored = { version = "3.0", optional = true }
 toml = {version = "0.5.8", optional = true}
 
 [features]
@@ -47,5 +47,5 @@ proto_gen = [
     "console",
     "toml"
 ]
-pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
+pb_gen = ["cmd_lib", "protoc-rust", "walkdir", "protoc-bin-vendored",]
 dart = ["proto_gen"]

shared-lib/lib-infra/src/pb.rs

@@ -7,6 +7,7 @@ use crate::proto_gen::*;
 use log::info;
 use std::fs::File;
 use std::io::Write;
+use std::path::PathBuf;
 use std::process::Command;
 use walkdir::WalkDir;
 
@@ -30,11 +31,14 @@ pub fn gen_files(crate_name: &str, root: &str) {
         }
     }
     println!("cargo:rerun-if-changed=build.rs");
+
+    let protoc_path = protoc_bin_vendored::protoc_bin_path().unwrap();
     #[cfg(feature = "dart")]
-    gen_pb_for_dart(crate_name, root, &paths, &file_names);
+    gen_pb_for_dart(crate_name, root, &paths, &file_names, &protoc_path);
 
     protoc_rust::Codegen::new()
         .out_dir("./src/protobuf/model")
+        .protoc_path(protoc_path)
         .inputs(&paths)
         .include(root)
         .run()
@@ -42,7 +46,7 @@ pub fn gen_files(crate_name: &str, root: &str) {
 }
 
 #[cfg(feature = "dart")]
-fn gen_pb_for_dart(name: &str, root: &str, paths: &Vec<String>, file_names: &Vec<String>) {
+fn gen_pb_for_dart(name: &str, root: &str, paths: &Vec<String>, file_names: &Vec<String>, proto_path: &PathBuf) {
     if std::env::var("CARGO_MAKE_WORKING_DIRECTORY").is_err() {
         log::warn!("CARGO_MAKE_WORKING_DIRECTORY was not set, skip generate dart pb");
         return;
@@ -59,17 +63,15 @@ fn gen_pb_for_dart(name: &str, root: &str, paths: &Vec<String>, file_names: &Vec
     if !std::path::Path::new(&output).exists() {
         std::fs::create_dir_all(&output).unwrap();
     }
-    check_pb_compiler();
-
     check_pb_dart_plugin();
-
+    let proto_path = proto_path.to_str().unwrap().to_owned();
     paths.iter().for_each(|path| {
         if cmd_lib::run_cmd! {
-            protoc --dart_out=${output} --proto_path=${root} ${path}
+            ${proto_path} --dart_out=${output} --proto_path=${root} ${path}
         }
         .is_err()
         {
-            panic!("Generate pb file failed with: {}", path)
+            panic!("Generate dart pb file failed with: {}", path)
         };
     });