feat: integrate postgres storage (#2604)

* chore: env config

* chore: get user workspace

* feat: enable postgres storage

* chore: add new env

* chore: add set env ffi

* chore: pass env before backend init

* chore: update

* fix: ci tests

* chore: commit the generate env file

* chore: remove unused import

frontend/rust-lib/flowy-database2/src/event_handler.rs

@@ -334,7 +334,7 @@ pub(crate) async fn create_row_handler(
   }
 }
 
-#[tracing::instrument(level = "trace", skip_all, err)]
+// #[tracing::instrument(level = "trace", skip_all, err)]
 pub(crate) async fn get_cell_handler(
   data: AFPluginData<CellIdPB>,
   manager: AFPluginState<Arc<DatabaseManager2>>,
@@ -560,7 +560,6 @@ pub(crate) async fn set_layout_setting_handler(
   Ok(())
 }
 
-#[tracing::instrument(level = "debug", skip(data, manager), err)]
 pub(crate) async fn get_layout_setting_handler(
   data: AFPluginData<DatabaseLayoutIdPB>,
   manager: AFPluginState<Arc<DatabaseManager2>>,

frontend/rust-lib/flowy-database2/src/manager.rs

@@ -3,7 +3,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 use appflowy_integrate::collab_builder::AppFlowyCollabBuilder;
-use appflowy_integrate::{RocksCollabDB, RocksDBConfig};
+use appflowy_integrate::{CollabPersistenceConfig, RocksCollabDB};
 use collab::core::collab::MutexCollab;
 use collab_database::database::DatabaseData;
 use collab_database::user::{UserDatabase as InnerUserDatabase, UserDatabaseCollabBuilder};
@@ -51,7 +51,7 @@ impl DatabaseManager2 {
     *self.user_database.lock() = Some(InnerUserDatabase::new(
       user_id,
       db,
-      RocksDBConfig::default(),
+      CollabPersistenceConfig::default(),
       UserDatabaseCollabBuilderImpl(self.collab_builder.clone()),
     ));
     // do nothing
@@ -229,7 +229,7 @@ impl UserDatabaseCollabBuilder for UserDatabaseCollabBuilderImpl {
     uid: i64,
     object_id: &str,
     db: Arc<RocksCollabDB>,
-    config: &RocksDBConfig,
+    config: &CollabPersistenceConfig,
   ) -> Arc<MutexCollab> {
     self.0.build_with_config(uid, object_id, db, config)
   }

frontend/rust-lib/flowy-database2/src/services/database_view/view_group.rs

@@ -3,8 +3,9 @@ use std::sync::Arc;
 use collab_database::fields::Field;
 use collab_database::rows::RowId;
 
-use flowy_error::FlowyResult;
+use flowy_error::{FlowyError, FlowyResult};
 use lib_infra::future::{to_fut, Fut};
+use tracing::trace;
 
 use crate::entities::FieldType;
 use crate::services::database_view::DatabaseViewData;
@@ -42,9 +43,10 @@ pub async fn new_group_controller(
   let fields = delegate.get_fields(&view_id, None).await;
   let rows = delegate.get_rows(&view_id).await;
   let layout = delegate.get_layout_for_view(&view_id);
+  trace!(?fields, ?rows, ?layout, "new_group_controller");
 
   // Read the grouping field or find a new grouping field
-  let grouping_field = setting_reader
+  let mut grouping_field = setting_reader
     .get_group_setting(&view_id)
     .await
     .and_then(|setting| {
@@ -52,17 +54,25 @@ pub async fn new_group_controller(
         .iter()
         .find(|field| field.id == setting.field_id)
         .cloned()
-    })
-    .unwrap_or_else(|| find_new_grouping_field(&fields, &layout).unwrap());
+    });
 
-  make_group_controller(
-    view_id,
-    grouping_field,
-    rows,
-    setting_reader,
-    setting_writer,
-  )
-  .await
+  if grouping_field.is_none() {
+    grouping_field = find_new_grouping_field(&fields, &layout);
+  }
+
+  match grouping_field {
+    None => Err(FlowyError::internal().context("No grouping field found".to_owned())),
+    Some(_) => {
+      make_group_controller(
+        view_id,
+        grouping_field.unwrap(),
+        rows,
+        setting_reader,
+        setting_writer,
+      )
+      .await
+    },
+  }
 }
 
 pub(crate) struct GroupSettingReaderImpl(pub Arc<dyn DatabaseViewData>);

frontend/rust-lib/flowy-database2/src/services/field/type_options/number_type_option/format.rs

@@ -7,15 +7,9 @@ use strum::IntoEnumIterator;
 use strum_macros::EnumIter;
 
 lazy_static! {
-  pub static ref CURRENCY_SYMBOL: Vec<String> = sorted_symbol();
-}
-
-fn sorted_symbol() -> Vec<String> {
-  let mut symbols = NumberFormat::iter()
+  pub static ref CURRENCY_SYMBOL: Vec<String> = NumberFormat::iter()
     .map(|format| format.symbol())
     .collect::<Vec<String>>();
-  symbols.sort_by(|a, b| b.len().cmp(&a.len()));
-  symbols
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, Serialize, Deserialize)]

frontend/rust-lib/flowy-database2/src/services/field/type_options/number_type_option/number_type_option.rs

@@ -137,9 +137,7 @@ impl NumberTypeOption {
           };
 
           match Decimal::from_str(&num_str) {
-            Ok(decimal, ..) => {
-              return Ok(NumberCellFormat::from_decimal(decimal));
-            },
+            Ok(decimal, ..) => Ok(NumberCellFormat::from_decimal(decimal)),
             Err(_) => Ok(NumberCellFormat::new()),
           }
         }

frontend/rust-lib/flowy-database2/src/services/field/type_options/number_type_option/number_type_option_entities.rs

@@ -27,7 +27,7 @@ impl NumberCellFormat {
       return Ok(Self::default());
     }
     // If the first char is not '-', then it is a sign.
-    let sign_positive = match num_str.find("-") {
+    let sign_positive = match num_str.find('-') {
       None => true,
       Some(offset) => offset != 0,
     };

frontend/rust-lib/flowy-database2/src/services/field/type_options/type_option_cell.rs

@@ -162,7 +162,7 @@ where
   ) {
     if let Some(cell_data_cache) = self.cell_data_cache.as_ref() {
       let field_type = FieldType::from(field.field_type);
-      let key = CellDataCacheKey::new(field, field_type.clone(), cell);
+      let key = CellDataCacheKey::new(field, field_type, cell);
       // tracing::trace!(
       //   "Cell cache update: field_type:{}, cell: {:?}, cell_data: {:?}",
       //   field_type,

frontend/rust-lib/flowy-database2/src/services/sort/controller.rs

@@ -85,7 +85,7 @@ impl SortController {
     self.gen_task(task_type, QualityOfService::Background).await;
   }
 
-  #[tracing::instrument(name = "process_sort_task", level = "debug", skip_all, err)]
+  // #[tracing::instrument(name = "process_sort_task", level = "trace", skip_all, err)]
   pub async fn process(&mut self, predicate: &str) -> FlowyResult<()> {
     let event_type = SortEvent::from_str(predicate).unwrap();
     let mut rows = self.delegate.get_rows(&self.view_id).await;

frontend/rust-lib/flowy-database2/tests/database/database_editor.rs

@@ -13,13 +13,13 @@ use flowy_database2::services::field::{
   SelectOptionCellChangeset, SingleSelectTypeOption,
 };
 use flowy_error::FlowyResult;
-use flowy_test::helper::ViewTest;
-use flowy_test::FlowySDKTest;
+use flowy_test::folder_event::ViewTest;
+use flowy_test::FlowyCoreTest;
 
 use crate::database::mock_data::{make_test_board, make_test_calendar, make_test_grid};
 
 pub struct DatabaseEditorTest {
-  pub sdk: FlowySDKTest,
+  pub sdk: FlowyCoreTest,
   pub app_id: String,
   pub view_id: String,
   pub editor: Arc<DatabaseEditor>,
@@ -43,7 +43,7 @@ impl DatabaseEditorTest {
   }
 
   pub async fn new(layout: DatabaseLayoutPB) -> Self {
-    let sdk = FlowySDKTest::default();
+    let sdk = FlowyCoreTest::new();
     let _ = sdk.init_user().await;
     let test = match layout {
       DatabaseLayoutPB::Grid => {

frontend/rust-lib/flowy-database2/tests/database/script.rs

@@ -1,425 +0,0 @@
-use bytes::Bytes;
-use database_model::entities::{
-  BuildGridContext, CellChangeset, Field, FieldChangesetParams, FieldMeta, FieldOrder, FieldType,
-  GridBlockInfoChangeset, GridBlockMetaSnapshot, InsertFieldParams, RowMeta, RowMetaChangeset,
-  RowOrder, TypeOptionDataFormat,
-};
-use flowy_client_sync::client_grid::GridBuilder;
-use flowy_database::services::field::*;
-use flowy_database::services::grid_meta_editor::{GridMetaEditor, GridPadBuilder};
-use flowy_database::services::row::CreateRowMetaPayload;
-use flowy_revision::REVISION_WRITE_INTERVAL_IN_MILLIS;
-use flowy_test::helper::ViewTest;
-use flowy_test::FlowySDKTest;
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::time::Duration;
-use strum::EnumCount;
-use tokio::time::sleep;
-
-pub enum EditorScript {
-  CreateField {
-    params: InsertFieldParams,
-  },
-  UpdateField {
-    changeset: FieldChangesetParams,
-  },
-  DeleteField {
-    field_meta: FieldMeta,
-  },
-  AssertFieldCount(usize),
-  AssertFieldEqual {
-    field_index: usize,
-    field_meta: FieldMeta,
-  },
-  CreateBlock {
-    block: GridBlockMetaSnapshot,
-  },
-  UpdateBlock {
-    changeset: GridBlockInfoChangeset,
-  },
-  AssertBlockCount(usize),
-  AssertBlock {
-    block_index: usize,
-    row_count: i32,
-    start_row_index: i32,
-  },
-  AssertBlockEqual {
-    block_index: usize,
-    block: GridBlockMetaSnapshot,
-  },
-  CreateEmptyRow,
-  CreateRow {
-    context: CreateRowMetaPayload,
-  },
-  UpdateRow {
-    changeset: RowMetaChangeset,
-  },
-  AssertRow {
-    changeset: RowMetaChangeset,
-  },
-  DeleteRow {
-    row_ids: Vec<String>,
-  },
-  UpdateCell {
-    changeset: CellChangeset,
-    is_err: bool,
-  },
-  AssertRowCount(usize),
-  // AssertRowEqual{ row_index: usize, row: RowMeta},
-  AssertGridMetaPad,
-}
-
-pub struct GridEditorTest {
-  pub sdk: FlowySDKTest,
-  pub grid_id: String,
-  pub editor: Arc<GridMetaEditor>,
-  pub field_metas: Vec<FieldMeta>,
-  pub grid_blocks: Vec<GridBlockMetaSnapshot>,
-  pub row_metas: Vec<Arc<RowMeta>>,
-  pub field_count: usize,
-
-  pub row_order_by_row_id: HashMap<String, RowOrder>,
-}
-
-impl GridEditorTest {
-  pub async fn new() -> Self {
-    let sdk = FlowySDKTest::default();
-    let _ = sdk.init_user().await;
-    let build_context = make_template_1_grid();
-    let view_data: Bytes = build_context.into();
-    let test = ViewTest::new_grid_view(&sdk, view_data.to_vec()).await;
-    let editor = sdk.grid_manager.open_grid(&test.view.id).await.unwrap();
-    let field_metas = editor.get_field_metas::<FieldOrder>(None).await.unwrap();
-    let grid_blocks = editor.get_block_metas().await.unwrap();
-    let row_metas = get_row_metas(&editor).await;
-
-    let grid_id = test.view.id;
-    Self {
-      sdk,
-      grid_id,
-      editor,
-      field_metas,
-      grid_blocks,
-      row_metas,
-      field_count: FieldType::COUNT,
-      row_order_by_row_id: HashMap::default(),
-    }
-  }
-
-  pub async fn run_scripts(&mut self, scripts: Vec<EditorScript>) {
-    for script in scripts {
-      self.run_script(script).await;
-    }
-  }
-
-  pub async fn run_script(&mut self, script: EditorScript) {
-    let grid_manager = self.sdk.grid_manager.clone();
-    let pool = self.sdk.user_session.db_pool().unwrap();
-    let rev_manager = self.editor.rev_manager();
-    let _cache = rev_manager.revision_cache().await;
-
-    match script {
-      EditorScript::CreateField { params } => {
-        if !self.editor.contain_field(&params.field.id).await {
-          self.field_count += 1;
-        }
-
-        self.editor.insert_field(params).await.unwrap();
-        self.field_metas = self
-          .editor
-          .get_field_metas::<FieldOrder>(None)
-          .await
-          .unwrap();
-        assert_eq!(self.field_count, self.field_metas.len());
-      },
-      EditorScript::UpdateField { changeset: change } => {
-        self.editor.update_field(change).await.unwrap();
-        self.field_metas = self
-          .editor
-          .get_field_metas::<FieldOrder>(None)
-          .await
-          .unwrap();
-      },
-      EditorScript::DeleteField { field_meta } => {
-        if self.editor.contain_field(&field_meta.id).await {
-          self.field_count -= 1;
-        }
-
-        self.editor.delete_field(&field_meta.id).await.unwrap();
-        self.field_metas = self
-          .editor
-          .get_field_metas::<FieldOrder>(None)
-          .await
-          .unwrap();
-        assert_eq!(self.field_count, self.field_metas.len());
-      },
-      EditorScript::AssertFieldCount(count) => {
-        assert_eq!(
-          self
-            .editor
-            .get_field_metas::<FieldOrder>(None)
-            .await
-            .unwrap()
-            .len(),
-          count
-        );
-      },
-      EditorScript::AssertFieldEqual {
-        field_index,
-        field_meta,
-      } => {
-        let field_metas = self
-          .editor
-          .get_field_metas::<FieldOrder>(None)
-          .await
-          .unwrap();
-        assert_eq!(field_metas[field_index].clone(), field_meta);
-      },
-      EditorScript::CreateBlock { block } => {
-        self.editor.create_block(block).await.unwrap();
-        self.grid_blocks = self.editor.get_block_metas().await.unwrap();
-      },
-      EditorScript::UpdateBlock { changeset: change } => {
-        self.editor.update_block(change).await.unwrap();
-      },
-      EditorScript::AssertBlockCount(count) => {
-        assert_eq!(self.editor.get_block_metas().await.unwrap().len(), count);
-      },
-      EditorScript::AssertBlock {
-        block_index,
-        row_count,
-        start_row_index,
-      } => {
-        assert_eq!(self.grid_blocks[block_index].row_count, row_count);
-        assert_eq!(
-          self.grid_blocks[block_index].start_row_index,
-          start_row_index
-        );
-      },
-      EditorScript::AssertBlockEqual { block_index, block } => {
-        let blocks = self.editor.get_block_metas().await.unwrap();
-        let compared_block = blocks[block_index].clone();
-        assert_eq!(compared_block, block);
-      },
-      EditorScript::CreateEmptyRow => {
-        let row_order = self.editor.create_row(None).await.unwrap();
-        self
-          .row_order_by_row_id
-          .insert(row_order.row_id.clone(), row_order);
-        self.row_metas = self.get_row_metas().await;
-        self.grid_blocks = self.editor.get_block_metas().await.unwrap();
-      },
-      EditorScript::CreateRow { context } => {
-        let row_orders = self.editor.insert_rows(vec![context]).await.unwrap();
-        for row_order in row_orders {
-          self
-            .row_order_by_row_id
-            .insert(row_order.row_id.clone(), row_order);
-        }
-        self.row_metas = self.get_row_metas().await;
-        self.grid_blocks = self.editor.get_block_metas().await.unwrap();
-      },
-      EditorScript::UpdateRow { changeset: change } => {
-        self.editor.update_row(change).await.unwrap()
-      },
-      EditorScript::DeleteRow { row_ids } => {
-        let row_orders = row_ids
-          .into_iter()
-          .map(|row_id| self.row_order_by_row_id.get(&row_id).unwrap().clone())
-          .collect::<Vec<RowOrder>>();
-
-        self.editor.delete_rows(row_orders).await.unwrap();
-        self.row_metas = self.get_row_metas().await;
-        self.grid_blocks = self.editor.get_block_metas().await.unwrap();
-      },
-      EditorScript::AssertRow { changeset } => {
-        let row = self
-          .row_metas
-          .iter()
-          .find(|row| row.id == changeset.row_id)
-          .unwrap();
-
-        if let Some(visibility) = changeset.visibility {
-          assert_eq!(row.visibility, visibility);
-        }
-
-        if let Some(height) = changeset.height {
-          assert_eq!(row.height, height);
-        }
-      },
-      EditorScript::UpdateCell { changeset, is_err } => {
-        let result = self.editor.update_cell(changeset).await;
-        if is_err {
-          assert!(result.is_err())
-        } else {
-          let _ = result.unwrap();
-          self.row_metas = self.get_row_metas().await;
-        }
-      },
-      EditorScript::AssertRowCount(count) => {
-        assert_eq!(self.row_metas.len(), count);
-      },
-      EditorScript::AssertGridMetaPad => {
-        sleep(Duration::from_millis(2 * REVISION_WRITE_INTERVAL_IN_MILLIS)).await;
-        let mut grid_rev_manager = grid_manager
-          .make_grid_rev_manager(&self.grid_id, pool.clone())
-          .unwrap();
-        let grid_pad = grid_rev_manager.load::<GridPadBuilder>(None).await.unwrap();
-        println!("{}", grid_pad.delta_str());
-      },
-    }
-  }
-
-  async fn get_row_metas(&self) -> Vec<Arc<RowMeta>> {
-    get_row_metas(&self.editor).await
-  }
-}
-
-async fn get_row_metas(editor: &Arc<GridMetaEditor>) -> Vec<Arc<RowMeta>> {
-  editor
-    .grid_block_snapshots(None)
-    .await
-    .unwrap()
-    .pop()
-    .unwrap()
-    .row_metas
-}
-
-pub fn create_text_field(grid_id: &str) -> (InsertFieldParams, FieldMeta) {
-  let field_meta = FieldBuilder::new(RichTextTypeOptionBuilder::default())
-    .name("Name")
-    .visibility(true)
-    .build();
-
-  let cloned_field_meta = field_meta.clone();
-
-  let type_option_data = field_meta
-    .get_type_option_entry::<RichTextTypeOptionPB>(&field_meta.field_type)
-    .unwrap()
-    .protobuf_bytes()
-    .to_vec();
-
-  let field = Field {
-    id: field_meta.id,
-    name: field_meta.name,
-    desc: field_meta.desc,
-    field_type: field_meta.field_type,
-    frozen: field_meta.frozen,
-    visibility: field_meta.visibility,
-    width: field_meta.width,
-    is_primary: false,
-  };
-
-  let params = InsertFieldParams {
-    grid_id: grid_id.to_owned(),
-    field,
-    type_option_data,
-    start_field_id: None,
-  };
-  (params, cloned_field_meta)
-}
-
-pub fn create_single_select_field(grid_id: &str) -> (InsertFieldParams, FieldMeta) {
-  let single_select = SingleSelectTypeOptionBuilder::default()
-    .option(SelectOption::new("Done"))
-    .option(SelectOption::new("Progress"));
-
-  let field_meta = FieldBuilder::new(single_select)
-    .name("Name")
-    .visibility(true)
-    .build();
-  let cloned_field_meta = field_meta.clone();
-  let type_option_data = field_meta
-    .get_type_option_entry::<SingleSelectTypeOption>(&field_meta.field_type)
-    .unwrap()
-    .protobuf_bytes()
-    .to_vec();
-
-  let field = Field {
-    id: field_meta.id,
-    name: field_meta.name,
-    desc: field_meta.desc,
-    field_type: field_meta.field_type,
-    frozen: field_meta.frozen,
-    visibility: field_meta.visibility,
-    width: field_meta.width,
-    is_primary: false,
-  };
-
-  let params = InsertFieldParams {
-    grid_id: grid_id.to_owned(),
-    field,
-    type_option_data,
-    start_field_id: None,
-  };
-  (params, cloned_field_meta)
-}
-
-fn make_template_1_grid() -> BuildGridContext {
-  let text_field = FieldBuilder::new(RichTextTypeOptionBuilder::default())
-    .name("Name")
-    .visibility(true)
-    .build();
-
-  // Single Select
-  let single_select = SingleSelectTypeOptionBuilder::default()
-    .option(SelectOption::new("Live"))
-    .option(SelectOption::new("Completed"))
-    .option(SelectOption::new("Planned"))
-    .option(SelectOption::new("Paused"));
-  let single_select_field = FieldBuilder::new(single_select)
-    .name("Status")
-    .visibility(true)
-    .build();
-
-  // MultiSelect
-  let multi_select = MultiSelectTypeOptionBuilder::default()
-    .option(SelectOption::new("Google"))
-    .option(SelectOption::new("Facebook"))
-    .option(SelectOption::new("Twitter"));
-  let multi_select_field = FieldBuilder::new(multi_select)
-    .name("Platform")
-    .visibility(true)
-    .build();
-
-  // Number
-  let number = NumberTypeOptionBuilder::default().set_format(NumberFormat::USD);
-  let number_field = FieldBuilder::new(number)
-    .name("Price")
-    .visibility(true)
-    .build();
-
-  // Date
-  let date = DateTypeOptionBuilder::default()
-    .date_format(DateFormat::US)
-    .time_format(TimeFormat::TwentyFourHour);
-  let date_field = FieldBuilder::new(date)
-    .name("Time")
-    .visibility(true)
-    .build();
-
-  // Checkbox
-  let checkbox = CheckboxTypeOptionBuilder::default();
-  let checkbox_field = FieldBuilder::new(checkbox)
-    .name("is done")
-    .visibility(true)
-    .build();
-
-  // URL
-  let url = URLTypeOptionBuilder::default();
-  let url_field = FieldBuilder::new(url).name("link").visibility(true).build();
-
-  GridBuilder::default()
-    .add_field(text_field)
-    .add_field(single_select_field)
-    .add_field(multi_select_field)
-    .add_field(number_field)
-    .add_field(date_field)
-    .add_field(checkbox_field)
-    .add_field(url_field)
-    .add_empty_row()
-    .add_empty_row()
-    .add_empty_row()
-    .build()
-}