feat: folder search mvp (#4665)

* feat: implement folder indexer

* feat: sqlite search views using fts5

* feat: add view indexing to user manager

* feat: implement folder indexer

* feat: add sqlite search documents

* feat: add document indexing to user manager

* feat: add document indexing to folder indexer

* chore: update collab rev

* feat: search frontend integration

* refactor: search index

* test: add event test

* chore: fix ci

* feat: initial command palette overlay impl (#4619)

* chore: test search engine

* chore: initial structure

* chore: replace old search request

* chore: enable log for lib-dispatch

* chore: move search manager to core

* feat: move traits and responsibility to search crate

* feat: move search to search crate

* feat: replace sqlite with tantivy

* feat: deserialize tantivy documents

* chore: fixes after rebase

* chore: clean code

* feat: fetch and sort results

* fix: code review + cleaning

* feat: support custom icons

* feat: support view layout icons

* feat: rename bloc and fix indexing

* fix: prettify dialog

* feat: score results

* chore: update collab rev

* feat: add recent view history to command palette

* test: add integration_tests

* fix: clippy changes

* fix: focus traversal in cmd palette

* fix: remove file after merging main

* chore: code review and panic-safe

* feat: index all views if index does not exist

* chore: improve logic with conditional

* chore: add is_empty check

* chore: abstract logic from folder manager init

* chore: update collab rev

* chore: code review

* chore: fixes after merge + update lock file

* chore: revert cargo lock

* fix: set icon type when removing icon

* fix: code review + dependency inversion

* fix: remove icon fix for not persisting icon type

* test: simple tests manipulating views

* test: create 100 views

* fix: tauri build

* chore: create 1000 views

* chore: create util methods

* chore: test

* chore: test

* chore: remove logs

* chore: fix build.rs

* chore: export models

* chore: enable clear cache on Rust-CI

* fix: navigate to newly created views

* fix: force disable setting workspace listener on rebuilds

* fix: remove late final

* fix: missing returns

* fix: localization and minor fixes

* test: add index assert to large test

* fix: missing section param after merging main

* chore: try fix unzip file error

* chore: lower the test

* feat: show hint when result is in trash

---------

Co-authored-by: nathan <nathan@appflowy.io>
Co-authored-by: Jiraffe7 <twajxjiraffe@gmail.com>
Co-authored-by: Lucas.Xu <lucas.xu@appflowy.io>

frontend/rust-lib/flowy-core/Cargo.toml

@@ -23,6 +23,7 @@ flowy-server-pub = { workspace = true }
 flowy-config = { workspace = true }
 flowy-date = { workspace = true }
 collab-integrate = { workspace = true }
+flowy-search = { workspace = true }
 collab-entity = { version = "0.1.0" }
 collab-plugins = { version = "0.1.0" }
 collab = { version = "0.1.0" }
@@ -35,7 +36,7 @@ tracing.workspace = true
 futures-core = { version = "0.3", default-features = false }
 bytes.workspace = true
 tokio = { workspace = true, features = ["full"] }
-tokio-stream = { workspace = true, features = ["sync"]}
+tokio-stream = { workspace = true, features = ["sync"] }
 console-subscriber = { version = "0.2", optional = true }
 parking_lot.workspace = true
 anyhow.workspace = true
@@ -56,14 +57,18 @@ http_sync = []
 native_sync = []
 use_bunyan = ["lib-log/use_bunyan"]
 dart = [
+    "flowy-user/dart",
+    "flowy-date/dart",
+    "flowy-search/dart",
     "flowy-folder/dart",
     "flowy-database2/dart",
 ]
 ts = [
     "flowy-user/tauri_ts",
     "flowy-folder/tauri_ts",
+    "flowy-search/tauri_ts",
     "flowy-database2/ts",
     "flowy-config/tauri_ts",
 ]
 rev-sqlite = ["flowy-user/rev-sqlite"]
-openssl_vendored = ["flowy-sqlite/openssl_vendored"]
+openssl_vendored = ["flowy-sqlite/openssl_vendored"]

frontend/rust-lib/flowy-core/src/config.rs

@@ -15,7 +15,7 @@ use crate::integrate::log::create_log_filter;
 pub struct AppFlowyCoreConfig {
   /// Different `AppFlowyCoreConfig` instance should have different name
   pub(crate) app_version: String,
-  pub(crate) name: String,
+  pub name: String,
   pub(crate) device_id: String,
   /// Used to store the user data
   pub storage_path: String,
@@ -102,8 +102,8 @@ impl AppFlowyCoreConfig {
     }
   }
 
-  pub fn log_filter(mut self, level: &str, with_crates: Vec<String>) -> Self {
-    self.log_filter = create_log_filter(level.to_owned(), with_crates);
+  pub fn log_filter(mut self, log_filter: String) -> Self {
+    self.log_filter = log_filter;
     self
   }
 }

frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs

@@ -1,4 +1,7 @@
 use bytes::Bytes;
+
+use tokio::sync::RwLock;
+
 use collab_integrate::collab_builder::AppFlowyCollabBuilder;
 use collab_integrate::CollabKVDB;
 use flowy_database2::entities::DatabaseLayoutPB;
@@ -14,10 +17,10 @@ use flowy_folder::manager::{FolderManager, FolderUser};
 use flowy_folder::share::ImportType;
 use flowy_folder::view_operation::{FolderOperationHandler, FolderOperationHandlers, View};
 use flowy_folder::ViewLayout;
+use flowy_search::folder::indexer::FolderIndexManagerImpl;
 use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::sync::{Arc, Weak};
-use tokio::sync::RwLock;
 
 use flowy_folder_pub::folder_builder::WorkspaceViewBuilder;
 use flowy_user::services::authenticate_user::AuthenticateUser;
@@ -35,6 +38,7 @@ impl FolderDepsResolver {
     database_manager: &Arc<DatabaseManager>,
     collab_builder: Arc<AppFlowyCollabBuilder>,
     server_provider: Arc<ServerProvider>,
+    folder_indexer: Arc<FolderIndexManagerImpl>,
   ) -> Arc<FolderManager> {
     let user: Arc<dyn FolderUser> = Arc::new(FolderUserImpl {
       authenticate_user: authenticate_user.clone(),
@@ -47,6 +51,7 @@ impl FolderDepsResolver {
         collab_builder,
         handlers,
         server_provider.clone(),
+        folder_indexer,
       )
       .await
       .unwrap(),

frontend/rust-lib/flowy-core/src/deps_resolve/mod.rs

@@ -2,12 +2,13 @@ pub use collab_deps::*;
 pub use database_deps::*;
 pub use document_deps::*;
 pub use folder_deps::*;
+pub use search_deps::*;
 pub use user_deps::*;
 
 mod collab_deps;
 mod document_deps;
 mod folder_deps;
-mod util;
 
 mod database_deps;
+mod search_deps;
 mod user_deps;

frontend/rust-lib/flowy-core/src/deps_resolve/search_deps.rs

@@ -0,0 +1,12 @@
+use flowy_search::folder::handler::FolderSearchHandler;
+use flowy_search::folder::indexer::FolderIndexManagerImpl;
+use flowy_search::services::manager::SearchManager;
+use std::sync::Arc;
+
+pub struct SearchDepsResolver();
+impl SearchDepsResolver {
+  pub async fn resolve(folder_indexer: Arc<FolderIndexManagerImpl>) -> Arc<SearchManager> {
+    let folder_handler = Arc::new(FolderSearchHandler::new(folder_indexer));
+    Arc::new(SearchManager::new(vec![folder_handler]))
+  }
+}

frontend/rust-lib/flowy-core/src/deps_resolve/util.rs

@@ -1 +0,0 @@
-

frontend/rust-lib/flowy-core/src/integrate/log.rs

@@ -12,7 +12,7 @@ pub(crate) fn init_log(config: &AppFlowyCoreConfig) {
       .build();
   }
 }
-pub(crate) fn create_log_filter(level: String, with_crates: Vec<String>) -> String {
+pub fn create_log_filter(level: String, with_crates: Vec<String>) -> String {
   let level = std::env::var("RUST_LOG").unwrap_or(level);
   let mut filters = with_crates
     .into_iter()
@@ -32,9 +32,13 @@ pub(crate) fn create_log_filter(level: String, with_crates: Vec<String>) -> Stri
   filters.push(format!("flowy_server={}", level));
   filters.push(format!("flowy_notification={}", "info"));
   filters.push(format!("lib_infra={}", level));
-  // filters.push(format!("lib_dispatch={}", level));
+  filters.push(format!("flowy_search={}", level));
+
+  // Most of the time, we don't need to see the logs from the following crates
+  // unless we are debugging the ffi or event dispatching
+  // filters.push(format!("lib_dispatch={}", level));
+  // filters.push(format!("dart_ffi={}", level));
 
-  filters.push(format!("dart_ffi={}", "info"));
   filters.push(format!("flowy_sqlite={}", "info"));
   filters.push(format!("client_api={}", level));
   #[cfg(feature = "profiling")]

frontend/rust-lib/flowy-core/src/integrate/mod.rs

@@ -1,5 +1,5 @@
 pub(crate) mod collab_interact;
-pub(crate) mod log;
+pub mod log;
 pub(crate) mod server;
 mod trait_impls;
 pub(crate) mod user;

frontend/rust-lib/flowy-core/src/lib.rs

@@ -1,5 +1,7 @@
 #![allow(unused_doc_comments)]
 
+use flowy_search::folder::indexer::FolderIndexManagerImpl;
+use flowy_search::services::manager::SearchManager;
 use flowy_storage::ObjectStorageService;
 use std::sync::Arc;
 use std::time::Duration;
@@ -11,6 +13,7 @@ use collab_integrate::collab_builder::{AppFlowyCollabBuilder, CollabPluginProvid
 use flowy_database2::DatabaseManager;
 use flowy_document::manager::DocumentManager;
 use flowy_folder::manager::FolderManager;
+
 use flowy_sqlite::kv::StorePreferences;
 use flowy_user::services::authenticate_user::AuthenticateUser;
 use flowy_user::services::entities::UserConfig;
@@ -30,7 +33,7 @@ use crate::integrate::user::UserStatusCallbackImpl;
 
 pub mod config;
 mod deps_resolve;
-mod integrate;
+pub mod integrate;
 pub mod module;
 
 /// This name will be used as to identify the current [AppFlowyCore] instance.
@@ -49,6 +52,7 @@ pub struct AppFlowyCore {
   pub server_provider: Arc<ServerProvider>,
   pub task_dispatcher: Arc<RwLock<TaskDispatcher>>,
   pub store_preference: Arc<StorePreferences>,
+  pub search_manager: Arc<SearchManager>,
 }
 
 impl AppFlowyCore {
@@ -102,6 +106,7 @@ impl AppFlowyCore {
       database_manager,
       document_manager,
       collab_builder,
+      search_manager,
     ) = async {
       /// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded
       /// on demand based on the [CollabPluginConfig].
@@ -141,17 +146,21 @@ impl AppFlowyCore {
         Arc::downgrade(&(server_provider.clone() as Arc<dyn ObjectStorageService>)),
       );
 
+      let folder_indexer = Arc::new(FolderIndexManagerImpl::new(Arc::downgrade(
+        &authenticate_user,
+      )));
       let folder_manager = FolderDepsResolver::resolve(
         Arc::downgrade(&authenticate_user),
         &document_manager,
         &database_manager,
         collab_builder.clone(),
         server_provider.clone(),
+        folder_indexer.clone(),
       )
       .await;
 
       let user_manager = UserDepsResolver::resolve(
-        authenticate_user,
+        authenticate_user.clone(),
         collab_builder.clone(),
         server_provider.clone(),
         store_preference.clone(),
@@ -160,6 +169,8 @@ impl AppFlowyCore {
       )
       .await;
 
+      let search_manager = SearchDepsResolver::resolve(folder_indexer).await;
+
       (
         user_manager,
         folder_manager,
@@ -167,6 +178,7 @@ impl AppFlowyCore {
         database_manager,
         document_manager,
         collab_builder,
+        search_manager,
       )
     }
     .await;
@@ -201,6 +213,7 @@ impl AppFlowyCore {
         Arc::downgrade(&database_manager),
         Arc::downgrade(&user_manager),
         Arc::downgrade(&document_manager),
+        Arc::downgrade(&search_manager),
       ),
     ));
 
@@ -214,6 +227,7 @@ impl AppFlowyCore {
       server_provider,
       task_dispatcher,
       store_preference,
+      search_manager,
     }
   }
 

frontend/rust-lib/flowy-core/src/module.rs

@@ -3,6 +3,7 @@ use std::sync::Weak;
 use flowy_database2::DatabaseManager;
 use flowy_document::manager::DocumentManager as DocumentManager2;
 use flowy_folder::manager::FolderManager;
+use flowy_search::services::manager::SearchManager;
 use flowy_user::user_manager::UserManager;
 use lib_dispatch::prelude::AFPlugin;
 
@@ -11,6 +12,7 @@ pub fn make_plugins(
   database_manager: Weak<DatabaseManager>,
   user_session: Weak<UserManager>,
   document_manager2: Weak<DocumentManager2>,
+  search_manager: Weak<SearchManager>,
 ) -> Vec<AFPlugin> {
   let store_preferences = user_session
     .upgrade()
@@ -22,6 +24,7 @@ pub fn make_plugins(
   let document_plugin2 = flowy_document::event_map::init(document_manager2);
   let config_plugin = flowy_config::event_map::init(store_preferences);
   let date_plugin = flowy_date::event_map::init();
+  let search_plugin = flowy_search::event_map::init(search_manager);
   vec![
     user_plugin,
     folder_plugin,
@@ -29,5 +32,6 @@ pub fn make_plugins(
     document_plugin2,
     config_plugin,
     date_plugin,
+    search_plugin,
   ]
 }