diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index 2052c1707c..2f07a518c8 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -1141,6 +1141,7 @@ dependencies = [ "bytes", "chrono", "crossbeam-utils", + "csv", "dashmap", "database-model", "diesel", diff --git a/frontend/rust-lib/flowy-database/Cargo.toml b/frontend/rust-lib/flowy-database/Cargo.toml index ec686c098e..06452bc74c 100644 --- a/frontend/rust-lib/flowy-database/Cargo.toml +++ b/frontend/rust-lib/flowy-database/Cargo.toml @@ -47,6 +47,7 @@ atomic_refcell = "0.1.9" crossbeam-utils = "0.8.15" async-stream = "0.3.4" parking_lot = "0.12.1" +csv = "1.1.6" [dev-dependencies] flowy-test = { path = "../flowy-test" } diff --git a/frontend/rust-lib/flowy-database/src/entities/database_entities.rs b/frontend/rust-lib/flowy-database/src/entities/database_entities.rs index 998b1db9b0..d28e0587f1 100644 --- a/frontend/rust-lib/flowy-database/src/entities/database_entities.rs +++ b/frontend/rust-lib/flowy-database/src/entities/database_entities.rs @@ -203,3 +203,9 @@ pub struct DatabaseLayoutIdPB { #[pb(index = 2)] pub layout: LayoutTypePB, } + +#[derive(Clone, ProtoBuf, Default, Debug)] +pub struct ExportCSVPB { + #[pb(index = 1)] + pub data: String, +} diff --git a/frontend/rust-lib/flowy-database/src/event_handler.rs b/frontend/rust-lib/flowy-database/src/event_handler.rs index d189c1f6fd..b2dbcda48a 100644 --- a/frontend/rust-lib/flowy-database/src/event_handler.rs +++ b/frontend/rust-lib/flowy-database/src/event_handler.rs @@ -1,6 +1,7 @@ use crate::entities::*; use crate::manager::DatabaseManager; use crate::services::cell::{FromCellString, ToCellChangesetString, TypeCellData}; +use crate::services::export::CSVExport; use crate::services::field::{ default_type_option_builder_from_type, select_type_option_from_field_rev, type_option_builder_from_json_str, DateCellChangeset, DateChangesetPB, SelectOptionCellChangeset, @@ -644,3 +645,16 @@ pub(crate) async fn get_calendar_event_handler( Some(event) => data_result_ok(event), } } + +#[tracing::instrument(level = "debug", skip(data, manager), err)] +pub(crate) async fn export_csv_handler( + data: AFPluginData, + manager: AFPluginState>, +) -> DataResult { + let params = data.into_inner(); + let database_editor = manager.get_database_editor(¶ms.value).await?; + let content = CSVExport + .export_database(¶ms.value, &database_editor) + .await?; + data_result_ok(ExportCSVPB { data: content }) +} diff --git a/frontend/rust-lib/flowy-database/src/event_map.rs b/frontend/rust-lib/flowy-database/src/event_map.rs index 401bcc3b19..dbe28d3ad2 100644 --- a/frontend/rust-lib/flowy-database/src/event_map.rs +++ b/frontend/rust-lib/flowy-database/src/event_map.rs @@ -55,7 +55,8 @@ pub fn init(database_manager: Arc) -> AFPlugin { .event(DatabaseEvent::GetCalendarEvent, get_calendar_event_handler) // Layout setting .event(DatabaseEvent::SetLayoutSetting, set_layout_setting_handler) - .event(DatabaseEvent::GetLayoutSetting, get_layout_setting_handler); + .event(DatabaseEvent::GetLayoutSetting, get_layout_setting_handler) + .event(DatabaseEvent::ExportCSV, export_csv_handler); plugin } @@ -256,4 +257,7 @@ pub enum DatabaseEvent { #[event(input = "MoveCalendarEventPB")] MoveCalendarEvent = 119, + + #[event(input = "DatabaseViewIdPB", output = "ExportCSVPB")] + ExportCSV = 120, } diff --git a/frontend/rust-lib/flowy-database/src/services/export.rs b/frontend/rust-lib/flowy-database/src/services/export.rs new file mode 100644 index 0000000000..da5c0728b3 --- /dev/null +++ b/frontend/rust-lib/flowy-database/src/services/export.rs @@ -0,0 +1,178 @@ +use crate::entities::FieldType; + +use crate::services::cell::TypeCellData; +use crate::services::database::DatabaseEditor; +use crate::services::field::{ + CheckboxTypeOptionPB, ChecklistTypeOptionPB, DateCellData, DateTypeOptionPB, + MultiSelectTypeOptionPB, NumberTypeOptionPB, RichTextTypeOptionPB, SingleSelectTypeOptionPB, + URLCellData, +}; +use database_model::{FieldRevision, TypeOptionDataDeserializer}; +use flowy_error::{FlowyError, FlowyResult}; +use indexmap::IndexMap; +use serde::Serialize; +use serde_json::{json, Map, Value}; +use std::collections::HashMap; + +use std::sync::Arc; + +#[derive(Debug, Clone, Serialize)] +pub struct ExportField { + pub id: String, + pub name: String, + pub field_type: i64, + pub visibility: bool, + pub width: i64, + pub type_options: HashMap, + pub is_primary: bool, +} + +#[derive(Debug, Clone, Serialize)] +struct ExportCell { + data: String, + field_type: FieldType, +} + +impl From<&Arc> for ExportField { + fn from(field_rev: &Arc) -> Self { + let field_type = FieldType::from(field_rev.ty); + let mut type_options: HashMap = HashMap::new(); + + field_rev.type_options.iter().for_each(|(k, s)| { + let value = match field_type { + FieldType::RichText => { + let pb = RichTextTypeOptionPB::from_json_str(s); + serde_json::to_value(pb).unwrap() + }, + FieldType::Number => { + let pb = NumberTypeOptionPB::from_json_str(s); + let mut map = Map::new(); + map.insert("format".to_string(), json!(pb.format as u8)); + map.insert("scale".to_string(), json!(pb.scale)); + map.insert("symbol".to_string(), json!(pb.symbol)); + map.insert("name".to_string(), json!(pb.name)); + Value::Object(map) + }, + FieldType::DateTime => { + let pb = DateTypeOptionPB::from_json_str(s); + let mut map = Map::new(); + map.insert("date_format".to_string(), json!(pb.date_format as u8)); + map.insert("time_format".to_string(), json!(pb.time_format as u8)); + map.insert("field_type".to_string(), json!(FieldType::DateTime as u8)); + Value::Object(map) + }, + FieldType::SingleSelect => { + let pb = SingleSelectTypeOptionPB::from_json_str(s); + let value = serde_json::to_string(&pb).unwrap(); + let mut map = Map::new(); + map.insert("content".to_string(), Value::String(value)); + Value::Object(map) + }, + FieldType::MultiSelect => { + let pb = MultiSelectTypeOptionPB::from_json_str(s); + let value = serde_json::to_string(&pb).unwrap(); + let mut map = Map::new(); + map.insert("content".to_string(), Value::String(value)); + Value::Object(map) + }, + FieldType::Checkbox => { + let pb = CheckboxTypeOptionPB::from_json_str(s); + serde_json::to_value(pb).unwrap() + }, + FieldType::URL => { + let pb = RichTextTypeOptionPB::from_json_str(s); + serde_json::to_value(pb).unwrap() + }, + FieldType::Checklist => { + let pb = ChecklistTypeOptionPB::from_json_str(s); + let value = serde_json::to_string(&pb).unwrap(); + let mut map = Map::new(); + map.insert("content".to_string(), Value::String(value)); + Value::Object(map) + }, + }; + type_options.insert(k.clone(), value); + }); + Self { + id: field_rev.id.clone(), + name: field_rev.name.clone(), + field_type: field_rev.ty as i64, + visibility: true, + width: 100, + type_options, + is_primary: field_rev.is_primary, + } + } +} + +pub struct CSVExport; +impl CSVExport { + pub async fn export_database( + &self, + view_id: &str, + database_editor: &Arc, + ) -> FlowyResult { + let mut wtr = csv::Writer::from_writer(vec![]); + let row_revs = database_editor.get_all_row_revs(view_id).await?; + let field_revs = database_editor.get_field_revs(None).await?; + + // Write fields + let field_records = field_revs + .iter() + .map(|field| ExportField::from(field)) + .map(|field| serde_json::to_string(&field).unwrap()) + .collect::>(); + + wtr + .write_record(&field_records) + .map_err(|e| FlowyError::internal().context(e))?; + + // Write rows + let mut field_by_field_id = IndexMap::new(); + field_revs.into_iter().for_each(|field| { + field_by_field_id.insert(field.id.clone(), field); + }); + for row_rev in row_revs { + let cells = field_by_field_id + .iter() + .map(|(field_id, field)| { + let field_type = FieldType::from(field.ty); + let data = row_rev + .cells + .get(field_id) + .map(|cell| TypeCellData::try_from(cell)) + .map(|data| { + data + .map(|data| match field_type { + FieldType::DateTime => { + match serde_json::from_str::(&data.cell_str) { + Ok(cell_data) => cell_data.timestamp.unwrap_or_default().to_string(), + Err(_) => "".to_string(), + } + }, + FieldType::URL => match serde_json::from_str::(&data.cell_str) { + Ok(cell_data) => cell_data.content, + Err(_) => "".to_string(), + }, + _ => data.cell_str, + }) + .unwrap_or_default() + }) + .unwrap_or_else(|| "".to_string()); + let cell = ExportCell { data, field_type }; + serde_json::to_string(&cell).unwrap() + }) + .collect::>(); + + if let Err(e) = wtr.write_record(&cells) { + tracing::warn!("CSV failed to write record: {}", e); + } + } + + let data = wtr + .into_inner() + .map_err(|e| FlowyError::internal().context(e))?; + let csv = String::from_utf8(data).map_err(|e| FlowyError::internal().context(e))?; + Ok(csv) + } +} diff --git a/frontend/rust-lib/flowy-database/src/services/field/type_options/date_type_option/date_type_option_entities.rs b/frontend/rust-lib/flowy-database/src/services/field/type_options/date_type_option/date_type_option_entities.rs index 1561252f87..60ee7a11a1 100644 --- a/frontend/rust-lib/flowy-database/src/services/field/type_options/date_type_option/date_type_option_entities.rs +++ b/frontend/rust-lib/flowy-database/src/services/field/type_options/date_type_option/date_type_option_entities.rs @@ -168,6 +168,7 @@ impl ToString for DateCellData { } #[derive(Clone, Debug, Copy, EnumIter, Serialize, Deserialize, ProtoBuf_Enum)] +#[repr(u8)] pub enum DateFormat { Local = 0, US = 1, @@ -216,6 +217,7 @@ impl DateFormat { #[derive( Clone, Copy, PartialEq, Eq, EnumIter, Debug, Hash, Serialize, Deserialize, ProtoBuf_Enum, )] +#[repr(u8)] pub enum TimeFormat { TwelveHour = 0, TwentyFourHour = 1, diff --git a/frontend/rust-lib/flowy-database/src/services/field/type_options/number_type_option/format.rs b/frontend/rust-lib/flowy-database/src/services/field/type_options/number_type_option/format.rs index 00d0c7ad8e..2eebcc93aa 100644 --- a/frontend/rust-lib/flowy-database/src/services/field/type_options/number_type_option/format.rs +++ b/frontend/rust-lib/flowy-database/src/services/field/type_options/number_type_option/format.rs @@ -14,6 +14,7 @@ lazy_static! { } #[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, Serialize, Deserialize, ProtoBuf_Enum)] +#[repr(u8)] pub enum NumberFormat { Num = 0, USD = 1, diff --git a/frontend/rust-lib/flowy-database/src/services/field/type_options/text_type_option/text_type_option.rs b/frontend/rust-lib/flowy-database/src/services/field/type_options/text_type_option/text_type_option.rs index 5f86f1fce3..e8e68f0758 100644 --- a/frontend/rust-lib/flowy-database/src/services/field/type_options/text_type_option/text_type_option.rs +++ b/frontend/rust-lib/flowy-database/src/services/field/type_options/text_type_option/text_type_option.rs @@ -37,7 +37,7 @@ impl TypeOptionBuilder for RichTextTypeOptionBuilder { pub struct RichTextTypeOptionPB { #[pb(index = 1)] #[serde(default)] - data: String, + pub data: String, } impl_type_option!(RichTextTypeOptionPB, FieldType::RichText); diff --git a/frontend/rust-lib/flowy-database/src/services/mod.rs b/frontend/rust-lib/flowy-database/src/services/mod.rs index a53e20fc06..529b118885 100644 --- a/frontend/rust-lib/flowy-database/src/services/mod.rs +++ b/frontend/rust-lib/flowy-database/src/services/mod.rs @@ -3,6 +3,7 @@ mod util; pub mod cell; pub mod database; pub mod database_view; +pub mod export; pub mod field; pub mod filter; pub mod group; diff --git a/frontend/rust-lib/flowy-database/tests/database/export_test.rs b/frontend/rust-lib/flowy-database/tests/database/export_test.rs new file mode 100644 index 0000000000..8fed6742ba --- /dev/null +++ b/frontend/rust-lib/flowy-database/tests/database/export_test.rs @@ -0,0 +1,23 @@ +use crate::database::database_editor::DatabaseEditorTest; +use flowy_database::services::export::CSVExport; + +#[tokio::test] +async fn export_test() { + let test = DatabaseEditorTest::new_grid().await; + + let s = CSVExport + .export_database(&test.view_id, &test.editor) + .await + .unwrap(); + + let mut reader = csv::Reader::from_reader(s.as_bytes()); + for header in reader.headers() { + println!("{:?}", header); + } + + let export_csv_records = reader.records(); + for record in export_csv_records { + let record = record.unwrap(); + println!("{:?}", record); + } +} diff --git a/frontend/rust-lib/flowy-database/tests/database/mod.rs b/frontend/rust-lib/flowy-database/tests/database/mod.rs index 0cb845d78c..f9bef0c795 100644 --- a/frontend/rust-lib/flowy-database/tests/database/mod.rs +++ b/frontend/rust-lib/flowy-database/tests/database/mod.rs @@ -9,4 +9,5 @@ mod layout_test; mod snapshot_test; mod sort_test; +mod export_test; mod mock_data;