chore: implement export handler (#2625)

* chore: implement export handler

* chore: fix export

frontend/rust-lib/Cargo.lock

@@ -1141,6 +1141,7 @@ dependencies = [
  "bytes",
  "chrono",
  "crossbeam-utils",
+ "csv",
  "dashmap",
  "database-model",
  "diesel",

frontend/rust-lib/flowy-database/Cargo.toml

@@ -47,6 +47,7 @@ atomic_refcell = "0.1.9"
 crossbeam-utils = "0.8.15"
 async-stream = "0.3.4"
 parking_lot = "0.12.1"
+csv = "1.1.6"
 
 [dev-dependencies]
 flowy-test = { path = "../flowy-test" }

frontend/rust-lib/flowy-database/src/entities/database_entities.rs

@@ -203,3 +203,9 @@ pub struct DatabaseLayoutIdPB {
   #[pb(index = 2)]
   pub layout: LayoutTypePB,
 }
+
+#[derive(Clone, ProtoBuf, Default, Debug)]
+pub struct ExportCSVPB {
+  #[pb(index = 1)]
+  pub data: String,
+}

frontend/rust-lib/flowy-database/src/event_handler.rs

@@ -1,6 +1,7 @@
 use crate::entities::*;
 use crate::manager::DatabaseManager;
 use crate::services::cell::{FromCellString, ToCellChangesetString, TypeCellData};
+use crate::services::export::CSVExport;
 use crate::services::field::{
   default_type_option_builder_from_type, select_type_option_from_field_rev,
   type_option_builder_from_json_str, DateCellChangeset, DateChangesetPB, SelectOptionCellChangeset,
@@ -644,3 +645,16 @@ pub(crate) async fn get_calendar_event_handler(
     Some(event) => data_result_ok(event),
   }
 }
+
+#[tracing::instrument(level = "debug", skip(data, manager), err)]
+pub(crate) async fn export_csv_handler(
+  data: AFPluginData<DatabaseViewIdPB>,
+  manager: AFPluginState<Arc<DatabaseManager>>,
+) -> DataResult<ExportCSVPB, FlowyError> {
+  let params = data.into_inner();
+  let database_editor = manager.get_database_editor(&params.value).await?;
+  let content = CSVExport
+    .export_database(&params.value, &database_editor)
+    .await?;
+  data_result_ok(ExportCSVPB { data: content })
+}

frontend/rust-lib/flowy-database/src/event_map.rs

@@ -55,7 +55,8 @@ pub fn init(database_manager: Arc<DatabaseManager>) -> AFPlugin {
         .event(DatabaseEvent::GetCalendarEvent, get_calendar_event_handler)
         // Layout setting
         .event(DatabaseEvent::SetLayoutSetting, set_layout_setting_handler)
-        .event(DatabaseEvent::GetLayoutSetting, get_layout_setting_handler);
+        .event(DatabaseEvent::GetLayoutSetting, get_layout_setting_handler)
+        .event(DatabaseEvent::ExportCSV, export_csv_handler);
 
   plugin
 }
@@ -256,4 +257,7 @@ pub enum DatabaseEvent {
 
   #[event(input = "MoveCalendarEventPB")]
   MoveCalendarEvent = 119,
+
+  #[event(input = "DatabaseViewIdPB", output = "ExportCSVPB")]
+  ExportCSV = 120,
 }

frontend/rust-lib/flowy-database/src/services/export.rs

@@ -0,0 +1,178 @@
+use crate::entities::FieldType;
+
+use crate::services::cell::TypeCellData;
+use crate::services::database::DatabaseEditor;
+use crate::services::field::{
+  CheckboxTypeOptionPB, ChecklistTypeOptionPB, DateCellData, DateTypeOptionPB,
+  MultiSelectTypeOptionPB, NumberTypeOptionPB, RichTextTypeOptionPB, SingleSelectTypeOptionPB,
+  URLCellData,
+};
+use database_model::{FieldRevision, TypeOptionDataDeserializer};
+use flowy_error::{FlowyError, FlowyResult};
+use indexmap::IndexMap;
+use serde::Serialize;
+use serde_json::{json, Map, Value};
+use std::collections::HashMap;
+
+use std::sync::Arc;
+
+#[derive(Debug, Clone, Serialize)]
+pub struct ExportField {
+  pub id: String,
+  pub name: String,
+  pub field_type: i64,
+  pub visibility: bool,
+  pub width: i64,
+  pub type_options: HashMap<String, Value>,
+  pub is_primary: bool,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct ExportCell {
+  data: String,
+  field_type: FieldType,
+}
+
+impl From<&Arc<FieldRevision>> for ExportField {
+  fn from(field_rev: &Arc<FieldRevision>) -> Self {
+    let field_type = FieldType::from(field_rev.ty);
+    let mut type_options: HashMap<String, Value> = HashMap::new();
+
+    field_rev.type_options.iter().for_each(|(k, s)| {
+      let value = match field_type {
+        FieldType::RichText => {
+          let pb = RichTextTypeOptionPB::from_json_str(s);
+          serde_json::to_value(pb).unwrap()
+        },
+        FieldType::Number => {
+          let pb = NumberTypeOptionPB::from_json_str(s);
+          let mut map = Map::new();
+          map.insert("format".to_string(), json!(pb.format as u8));
+          map.insert("scale".to_string(), json!(pb.scale));
+          map.insert("symbol".to_string(), json!(pb.symbol));
+          map.insert("name".to_string(), json!(pb.name));
+          Value::Object(map)
+        },
+        FieldType::DateTime => {
+          let pb = DateTypeOptionPB::from_json_str(s);
+          let mut map = Map::new();
+          map.insert("date_format".to_string(), json!(pb.date_format as u8));
+          map.insert("time_format".to_string(), json!(pb.time_format as u8));
+          map.insert("field_type".to_string(), json!(FieldType::DateTime as u8));
+          Value::Object(map)
+        },
+        FieldType::SingleSelect => {
+          let pb = SingleSelectTypeOptionPB::from_json_str(s);
+          let value = serde_json::to_string(&pb).unwrap();
+          let mut map = Map::new();
+          map.insert("content".to_string(), Value::String(value));
+          Value::Object(map)
+        },
+        FieldType::MultiSelect => {
+          let pb = MultiSelectTypeOptionPB::from_json_str(s);
+          let value = serde_json::to_string(&pb).unwrap();
+          let mut map = Map::new();
+          map.insert("content".to_string(), Value::String(value));
+          Value::Object(map)
+        },
+        FieldType::Checkbox => {
+          let pb = CheckboxTypeOptionPB::from_json_str(s);
+          serde_json::to_value(pb).unwrap()
+        },
+        FieldType::URL => {
+          let pb = RichTextTypeOptionPB::from_json_str(s);
+          serde_json::to_value(pb).unwrap()
+        },
+        FieldType::Checklist => {
+          let pb = ChecklistTypeOptionPB::from_json_str(s);
+          let value = serde_json::to_string(&pb).unwrap();
+          let mut map = Map::new();
+          map.insert("content".to_string(), Value::String(value));
+          Value::Object(map)
+        },
+      };
+      type_options.insert(k.clone(), value);
+    });
+    Self {
+      id: field_rev.id.clone(),
+      name: field_rev.name.clone(),
+      field_type: field_rev.ty as i64,
+      visibility: true,
+      width: 100,
+      type_options,
+      is_primary: field_rev.is_primary,
+    }
+  }
+}
+
+pub struct CSVExport;
+impl CSVExport {
+  pub async fn export_database(
+    &self,
+    view_id: &str,
+    database_editor: &Arc<DatabaseEditor>,
+  ) -> FlowyResult<String> {
+    let mut wtr = csv::Writer::from_writer(vec![]);
+    let row_revs = database_editor.get_all_row_revs(view_id).await?;
+    let field_revs = database_editor.get_field_revs(None).await?;
+
+    // Write fields
+    let field_records = field_revs
+      .iter()
+      .map(|field| ExportField::from(field))
+      .map(|field| serde_json::to_string(&field).unwrap())
+      .collect::<Vec<String>>();
+
+    wtr
+      .write_record(&field_records)
+      .map_err(|e| FlowyError::internal().context(e))?;
+
+    // Write rows
+    let mut field_by_field_id = IndexMap::new();
+    field_revs.into_iter().for_each(|field| {
+      field_by_field_id.insert(field.id.clone(), field);
+    });
+    for row_rev in row_revs {
+      let cells = field_by_field_id
+        .iter()
+        .map(|(field_id, field)| {
+          let field_type = FieldType::from(field.ty);
+          let data = row_rev
+            .cells
+            .get(field_id)
+            .map(|cell| TypeCellData::try_from(cell))
+            .map(|data| {
+              data
+                .map(|data| match field_type {
+                  FieldType::DateTime => {
+                    match serde_json::from_str::<DateCellData>(&data.cell_str) {
+                      Ok(cell_data) => cell_data.timestamp.unwrap_or_default().to_string(),
+                      Err(_) => "".to_string(),
+                    }
+                  },
+                  FieldType::URL => match serde_json::from_str::<URLCellData>(&data.cell_str) {
+                    Ok(cell_data) => cell_data.content,
+                    Err(_) => "".to_string(),
+                  },
+                  _ => data.cell_str,
+                })
+                .unwrap_or_default()
+            })
+            .unwrap_or_else(|| "".to_string());
+          let cell = ExportCell { data, field_type };
+          serde_json::to_string(&cell).unwrap()
+        })
+        .collect::<Vec<_>>();
+
+      if let Err(e) = wtr.write_record(&cells) {
+        tracing::warn!("CSV failed to write record: {}", e);
+      }
+    }
+
+    let data = wtr
+      .into_inner()
+      .map_err(|e| FlowyError::internal().context(e))?;
+    let csv = String::from_utf8(data).map_err(|e| FlowyError::internal().context(e))?;
+    Ok(csv)
+  }
+}

frontend/rust-lib/flowy-database/src/services/field/type_options/date_type_option/date_type_option_entities.rs

@@ -168,6 +168,7 @@ impl ToString for DateCellData {
 }
 
 #[derive(Clone, Debug, Copy, EnumIter, Serialize, Deserialize, ProtoBuf_Enum)]
+#[repr(u8)]
 pub enum DateFormat {
   Local = 0,
   US = 1,
@@ -216,6 +217,7 @@ impl DateFormat {
 #[derive(
   Clone, Copy, PartialEq, Eq, EnumIter, Debug, Hash, Serialize, Deserialize, ProtoBuf_Enum,
 )]
+#[repr(u8)]
 pub enum TimeFormat {
   TwelveHour = 0,
   TwentyFourHour = 1,

frontend/rust-lib/flowy-database/src/services/field/type_options/number_type_option/format.rs

@@ -14,6 +14,7 @@ lazy_static! {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, Serialize, Deserialize, ProtoBuf_Enum)]
+#[repr(u8)]
 pub enum NumberFormat {
   Num = 0,
   USD = 1,

frontend/rust-lib/flowy-database/src/services/field/type_options/text_type_option/text_type_option.rs

@@ -37,7 +37,7 @@ impl TypeOptionBuilder for RichTextTypeOptionBuilder {
 pub struct RichTextTypeOptionPB {
   #[pb(index = 1)]
   #[serde(default)]
-  data: String,
+  pub data: String,
 }
 impl_type_option!(RichTextTypeOptionPB, FieldType::RichText);
 

frontend/rust-lib/flowy-database/src/services/mod.rs

@@ -3,6 +3,7 @@ mod util;
 pub mod cell;
 pub mod database;
 pub mod database_view;
+pub mod export;
 pub mod field;
 pub mod filter;
 pub mod group;

frontend/rust-lib/flowy-database/tests/database/export_test.rs

@@ -0,0 +1,23 @@
+use crate::database::database_editor::DatabaseEditorTest;
+use flowy_database::services::export::CSVExport;
+
+#[tokio::test]
+async fn export_test() {
+  let test = DatabaseEditorTest::new_grid().await;
+
+  let s = CSVExport
+    .export_database(&test.view_id, &test.editor)
+    .await
+    .unwrap();
+
+  let mut reader = csv::Reader::from_reader(s.as_bytes());
+  for header in reader.headers() {
+    println!("{:?}", header);
+  }
+
+  let export_csv_records = reader.records();
+  for record in export_csv_records {
+    let record = record.unwrap();
+    println!("{:?}", record);
+  }
+}

frontend/rust-lib/flowy-database/tests/database/mod.rs

@@ -9,4 +9,5 @@ mod layout_test;
 mod snapshot_test;
 mod sort_test;
 
+mod export_test;
 mod mock_data;