From 041e3c155af1c9685caec1c347cc183e3f4389bf Mon Sep 17 00:00:00 2001 From: Mathias Mogensen <42929161+Xazin@users.noreply.github.com> Date: Wed, 5 Jun 2024 04:05:51 +0200 Subject: [PATCH] fix: search workspace sync indexing (#5437) * fix: search workspace sync indexing * chore: update collab rev temporarily * feat: revert comparison and implement index check * chore: fixes after merg * chore: enable search * chore: disable ai test --------- Co-authored-by: nathan --- .../lib/shared/feature_flags.dart | 2 +- .../lib/startup/tasks/app_widget.dart | 12 +- .../action_navigation_bloc.dart | 4 +- .../command_palette/command_palette_bloc.dart | 13 +- .../home/menu/sidebar/sidebar.dart | 5 +- frontend/appflowy_tauri/src-tauri/Cargo.lock | 2 + frontend/appflowy_web/wasm-libs/Cargo.lock | 240 +++++++++++++++++- .../appflowy_web_app/src-tauri/Cargo.lock | 1 + frontend/rust-lib/Cargo.lock | 2 + .../tests/chat/chat_message_test.rs | 58 ++--- .../src/deps_resolve/folder_deps.rs | 4 +- .../rust-lib/flowy-core/src/integrate/user.rs | 2 +- frontend/rust-lib/flowy-core/src/lib.rs | 6 +- frontend/rust-lib/flowy-folder/Cargo.toml | 3 +- .../flowy-folder/src/entities/workspace.rs | 5 + frontend/rust-lib/flowy-folder/src/manager.rs | 6 +- .../rust-lib/flowy-folder/src/manager_init.rs | 54 +++- .../rust-lib/flowy-search-pub/src/entities.rs | 12 + .../flowy-search/src/folder/indexer.rs | 10 +- 19 files changed, 368 insertions(+), 73 deletions(-) diff --git a/frontend/appflowy_flutter/lib/shared/feature_flags.dart b/frontend/appflowy_flutter/lib/shared/feature_flags.dart index 94b45da631..9f578c177a 100644 --- a/frontend/appflowy_flutter/lib/shared/feature_flags.dart +++ b/frontend/appflowy_flutter/lib/shared/feature_flags.dart @@ -102,9 +102,9 @@ enum FeatureFlag { switch (this) { case FeatureFlag.collaborativeWorkspace: case FeatureFlag.membersSettings: - case FeatureFlag.search: case FeatureFlag.unknown: return false; + case FeatureFlag.search: case FeatureFlag.syncDocument: case FeatureFlag.syncDatabase: return true; diff --git a/frontend/appflowy_flutter/lib/startup/tasks/app_widget.dart b/frontend/appflowy_flutter/lib/startup/tasks/app_widget.dart index f90685f030..25b5b419b5 100644 --- a/frontend/appflowy_flutter/lib/startup/tasks/app_widget.dart +++ b/frontend/appflowy_flutter/lib/startup/tasks/app_widget.dart @@ -1,5 +1,8 @@ import 'dart:io'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; + import 'package:appflowy/mobile/application/mobile_router.dart'; import 'package:appflowy/plugins/document/application/document_appearance_cubit.dart'; import 'package:appflowy/shared/feature_flags.dart'; @@ -13,6 +16,7 @@ import 'package:appflowy/workspace/application/notification/notification_service import 'package:appflowy/workspace/application/settings/appearance/appearance_cubit.dart'; import 'package:appflowy/workspace/application/settings/notifications/notification_settings_cubit.dart'; import 'package:appflowy/workspace/application/sidebar/rename_view/rename_view_bloc.dart'; +import 'package:appflowy/workspace/application/tabs/tabs_bloc.dart'; import 'package:appflowy/workspace/application/view/view_ext.dart'; import 'package:appflowy/workspace/presentation/command_palette/command_palette.dart'; import 'package:appflowy_backend/log.dart'; @@ -21,8 +25,6 @@ import 'package:appflowy_editor/appflowy_editor.dart' hide Log; import 'package:easy_localization/easy_localization.dart'; import 'package:flowy_infra/theme.dart'; import 'package:flowy_infra_ui/flowy_infra_ui.dart'; -import 'package:flutter/material.dart'; -import 'package:flutter/services.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:go_router/go_router.dart'; @@ -175,8 +177,12 @@ class _ApplicationWidgetState extends State { if (action?.type == ActionType.openView && PlatformExtension.isDesktop) { final view = action!.arguments?[ActionArgumentKeys.view]; + final nodePath = action.arguments?[ActionArgumentKeys.nodePath]; if (view != null) { - AppGlobals.rootNavKey.currentContext?.pushView(view); + getIt().openPlugin( + view.plugin(), + arguments: {PluginArgumentKeys.selection: nodePath}, + ); } } else if (action?.type == ActionType.openRow && PlatformExtension.isMobile) { diff --git a/frontend/appflowy_flutter/lib/workspace/application/action_navigation/action_navigation_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/action_navigation/action_navigation_bloc.dart index 467115368d..5bc64b6785 100644 --- a/frontend/appflowy_flutter/lib/workspace/application/action_navigation/action_navigation_bloc.dart +++ b/frontend/appflowy_flutter/lib/workspace/application/action_navigation/action_navigation_bloc.dart @@ -21,11 +21,11 @@ class ActionNavigationBloc currentAction = currentAction.copyWith(arguments: {}); } - action.arguments?.addAll({ActionArgumentKeys.view: view}); + currentAction.arguments?.addAll({ActionArgumentKeys.view: view}); } } - emit(state.copyWith(action: action, nextActions: nextActions)); + emit(state.copyWith(action: currentAction, nextActions: nextActions)); if (nextActions.isNotEmpty) { final newActions = [...nextActions]; diff --git a/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart index 0c45cd8bf4..03a35836e9 100644 --- a/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart +++ b/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart @@ -101,22 +101,15 @@ class CommandPaletteBloc Future _initTrash() async { _trashListener.start( trashUpdated: (trashOrFailed) { - final trash = trashOrFailed.fold( - (trash) => trash, - (error) => null, - ); - + final trash = trashOrFailed.toNullable(); add(CommandPaletteEvent.trashChanged(trash: trash)); }, ); final trashOrFailure = await _trashService.readTrash(); - final trashRes = trashOrFailure.fold( - (trash) => trash, - (error) => null, - ); + final trash = trashOrFailure.toNullable(); - add(CommandPaletteEvent.trashChanged(trash: trashRes?.items)); + add(CommandPaletteEvent.trashChanged(trash: trash?.items)); } void _debounceOnSearchChanged(String value) { diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/sidebar.dart b/frontend/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/sidebar.dart index 6956bc51d6..fe8b0db70b 100644 --- a/frontend/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/sidebar.dart +++ b/frontend/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/sidebar.dart @@ -1,5 +1,7 @@ import 'dart:async'; +import 'package:flutter/material.dart'; + import 'package:appflowy/generated/flowy_svgs.g.dart'; import 'package:appflowy/generated/locale_keys.g.dart'; import 'package:appflowy/shared/feature_flags.dart'; @@ -30,7 +32,6 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flowy_infra_ui/style_widget/button.dart'; import 'package:flowy_infra_ui/style_widget/text.dart'; import 'package:flowy_infra_ui/widget/spacing.dart'; -import 'package:flutter/material.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; /// Home Sidebar is the left side bar of the home page. @@ -95,7 +96,7 @@ class HomeSideBar extends StatelessWidget { } return MultiBlocProvider( providers: [ - BlocProvider(create: (_) => getIt()), + BlocProvider.value(value: getIt()), BlocProvider( create: (_) => SidebarSectionsBloc() ..add( diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.lock b/frontend/appflowy_tauri/src-tauri/Cargo.lock index 0c48e8f203..6a94085951 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.lock +++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock @@ -2129,12 +2129,14 @@ dependencies = [ "flowy-folder-pub", "flowy-notification", "flowy-search-pub", + "flowy-sqlite", "lazy_static", "lib-dispatch", "lib-infra", "nanoid", "parking_lot 0.12.1", "protobuf", + "serde", "serde_json", "strum_macros 0.21.1", "tokio", diff --git a/frontend/appflowy_web/wasm-libs/Cargo.lock b/frontend/appflowy_web/wasm-libs/Cargo.lock index b8da195c3d..d93af21f59 100644 --- a/frontend/appflowy_web/wasm-libs/Cargo.lock +++ b/frontend/appflowy_web/wasm-libs/Cargo.lock @@ -1004,6 +1004,41 @@ dependencies = [ "cipher", ] +[[package]] +name = "darling" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.48", +] + +[[package]] +name = "darling_macro" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.48", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -1091,6 +1126,53 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae2a35373c5c74340b79ae6780b498b2b183915ec5dacf263aac5a099bf485a" +[[package]] +name = "diesel" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b696af9ff4c0d2a507db2c5faafa8aa0205e297e5f11e203a24226d5355e7a" +dependencies = [ + "chrono", + "diesel_derives", + "libsqlite3-sys", + "r2d2", + "serde_json", + "time", +] + +[[package]] +name = "diesel_derives" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6fdd83d5947068817016e939596d246e5367279453f2a3433287894f2f2996" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "diesel_migrations" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" +dependencies = [ + "syn 2.0.48", +] + [[package]] name = "digest" version = "0.10.7" @@ -1102,6 +1184,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "dsl_auto_type" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab32c18ea6760d951659768a3e35ea72fc1ba0916d665a88dfe048b2a41e543f" +dependencies = [ + "darling", + "either", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "dtoa" version = "1.0.9" @@ -1286,7 +1382,7 @@ dependencies = [ "similar 1.3.0", "syn 1.0.109", "tera", - "toml", + "toml 0.5.11", "walkdir", ] @@ -1421,12 +1517,14 @@ dependencies = [ "flowy-folder-pub", "flowy-notification", "flowy-search-pub", + "flowy-sqlite", "lazy_static", "lib-dispatch", "lib-infra", "nanoid", "parking_lot 0.12.1", "protobuf", + "serde", "serde_json", "strum_macros 0.21.1", "tokio", @@ -1532,6 +1630,24 @@ dependencies = [ "serde_repr", ] +[[package]] +name = "flowy-sqlite" +version = "0.1.0" +dependencies = [ + "anyhow", + "diesel", + "diesel_derives", + "diesel_migrations", + "libsqlite3-sys", + "parking_lot 0.12.1", + "r2d2", + "scheduled-thread-pool", + "serde", + "serde_json", + "thiserror", + "tracing", +] + [[package]] name = "flowy-storage" version = "0.1.0" @@ -1883,6 +1999,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.4" @@ -2050,6 +2172,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.3.0" @@ -2321,6 +2449,17 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.14" @@ -2439,6 +2578,27 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" +[[package]] +name = "migrations_internals" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" +dependencies = [ + "serde", + "toml 0.8.14", +] + +[[package]] +name = "migrations_macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + [[package]] name = "mime" version = "0.3.17" @@ -3237,6 +3397,17 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot 0.12.1", + "scheduled-thread-pool", +] + [[package]] name = "rand" version = "0.7.3" @@ -3545,6 +3716,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot 0.12.1", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3712,6 +3892,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3917,6 +4106,12 @@ dependencies = [ "quote", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum_macros" version = "0.21.1" @@ -4254,6 +4449,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -4943,6 +5172,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c52728401e1dc672a56e81e593e912aa54c78f40246869f78359a2bf24d29d" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/frontend/appflowy_web_app/src-tauri/Cargo.lock b/frontend/appflowy_web_app/src-tauri/Cargo.lock index d7aaa79525..2ba17e8683 100644 --- a/frontend/appflowy_web_app/src-tauri/Cargo.lock +++ b/frontend/appflowy_web_app/src-tauri/Cargo.lock @@ -2166,6 +2166,7 @@ dependencies = [ "flowy-folder-pub", "flowy-notification", "flowy-search-pub", + "flowy-sqlite", "lazy_static", "lib-dispatch", "lib-infra", diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index b98f682a2c..e3570807f0 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -1968,12 +1968,14 @@ dependencies = [ "flowy-folder-pub", "flowy-notification", "flowy-search-pub", + "flowy-sqlite", "lazy_static", "lib-dispatch", "lib-infra", "nanoid", "parking_lot 0.12.1", "protobuf", + "serde", "serde_json", "strum_macros 0.21.1", "tokio", diff --git a/frontend/rust-lib/event-integration-test/tests/chat/chat_message_test.rs b/frontend/rust-lib/event-integration-test/tests/chat/chat_message_test.rs index 1efe4c77d0..8594193ab6 100644 --- a/frontend/rust-lib/event-integration-test/tests/chat/chat_message_test.rs +++ b/frontend/rust-lib/event-integration-test/tests/chat/chat_message_test.rs @@ -1,7 +1,7 @@ use crate::util::receive_with_timeout; use event_integration_test::user_event::user_localhost_af_cloud; use event_integration_test::EventIntegrationTest; -use flowy_chat::entities::{ChatMessageListPB, ChatMessageTypePB}; +use flowy_chat::entities::ChatMessageListPB; use flowy_chat::notification::ChatNotification; use flowy_chat_pub::cloud::ChatMessageType; @@ -131,31 +131,31 @@ async fn af_cloud_load_remote_system_message_test() { assert_eq!(first_five_messages.messages[4].content, "hello server 0"); } -#[tokio::test] -async fn af_cloud_load_remote_user_message_test() { - user_localhost_af_cloud().await; - let test = EventIntegrationTest::new().await; - test.af_cloud_sign_up().await; - - let current_workspace = test.get_current_workspace().await; - let view = test.create_chat(¤t_workspace.id).await; - let chat_id = view.id.clone(); - let rx = test - .notification_sender - .subscribe_without_payload(&chat_id, ChatNotification::FinishAnswerQuestion); - test - .send_message(&chat_id, "hello world", ChatMessageTypePB::User) - .await; - let _ = receive_with_timeout(rx, Duration::from_secs(60)) - .await - .unwrap(); - - let all = test.load_next_message(&chat_id, 5, None).await; - assert_eq!(all.messages.len(), 2); - // 3 means AI - assert_eq!(all.messages[0].author_type, 3); - // 2 means User - assert_eq!(all.messages[1].author_type, 1); - // The message ID is incremented by 1. - assert_eq!(all.messages[1].message_id + 1, all.messages[0].message_id); -} +// #[tokio::test] +// async fn af_cloud_load_remote_user_message_test() { +// user_localhost_af_cloud().await; +// let test = EventIntegrationTest::new().await; +// test.af_cloud_sign_up().await; +// +// let current_workspace = test.get_current_workspace().await; +// let view = test.create_chat(¤t_workspace.id).await; +// let chat_id = view.id.clone(); +// let rx = test +// .notification_sender +// .subscribe_without_payload(&chat_id, ChatNotification::FinishAnswerQuestion); +// test +// .send_message(&chat_id, "hello world", ChatMessageTypePB::User) +// .await; +// let _ = receive_with_timeout(rx, Duration::from_secs(60)) +// .await +// .unwrap(); +// +// let all = test.load_next_message(&chat_id, 5, None).await; +// assert_eq!(all.messages.len(), 2); +// // 3 means AI +// assert_eq!(all.messages[0].author_type, 3); +// // 2 means User +// assert_eq!(all.messages[1].author_type, 1); +// // The message ID is incremented by 1. +// assert_eq!(all.messages[1].message_id + 1, all.messages[0].message_id); +// } diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs index 92e09996d5..e826a3f4c4 100644 --- a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs @@ -18,6 +18,7 @@ use flowy_folder::view_operation::{ use flowy_folder::ViewLayout; use flowy_folder_pub::folder_builder::NestedViewBuilder; use flowy_search::folder::indexer::FolderIndexManagerImpl; +use flowy_sqlite::kv::StorePreferences; use flowy_user::services::authenticate_user::AuthenticateUser; use lib_dispatch::prelude::ToBytes; use lib_infra::future::FutureResult; @@ -37,6 +38,7 @@ impl FolderDepsResolver { collab_builder: Arc, server_provider: Arc, folder_indexer: Arc, + store_preferences: Arc, chat_manager: &Arc, ) -> Arc { let user: Arc = Arc::new(FolderUserImpl { @@ -55,8 +57,8 @@ impl FolderDepsResolver { handlers, server_provider.clone(), folder_indexer, + store_preferences, ) - .await .unwrap(), ) } diff --git a/frontend/rust-lib/flowy-core/src/integrate/user.rs b/frontend/rust-lib/flowy-core/src/integrate/user.rs index b68e54f2dc..f8219cdaff 100644 --- a/frontend/rust-lib/flowy-core/src/integrate/user.rs +++ b/frontend/rust-lib/flowy-core/src/integrate/user.rs @@ -1,9 +1,9 @@ use std::sync::Arc; use anyhow::Context; -use collab_entity::CollabType; use tracing::event; +use collab_entity::CollabType; use collab_integrate::collab_builder::AppFlowyCollabBuilder; use flowy_database2::DatabaseManager; use flowy_document::manager::DocumentManager; diff --git a/frontend/rust-lib/flowy-core/src/lib.rs b/frontend/rust-lib/flowy-core/src/lib.rs index 80bb1213f5..02f63c6f1d 100644 --- a/frontend/rust-lib/flowy-core/src/lib.rs +++ b/frontend/rust-lib/flowy-core/src/lib.rs @@ -169,7 +169,10 @@ impl AppFlowyCore { let chat_manager = ChatDepsResolver::resolve(Arc::downgrade(&authenticate_user), server_provider.clone()); - let folder_indexer = Arc::new(FolderIndexManagerImpl::new(None)); + + let folder_indexer = Arc::new(FolderIndexManagerImpl::new(Some(Arc::downgrade( + &authenticate_user, + )))); let folder_manager = FolderDepsResolver::resolve( Arc::downgrade(&authenticate_user), &document_manager, @@ -177,6 +180,7 @@ impl AppFlowyCore { collab_builder.clone(), server_provider.clone(), folder_indexer.clone(), + store_preference.clone(), &chat_manager, ) .await; diff --git a/frontend/rust-lib/flowy-folder/Cargo.toml b/frontend/rust-lib/flowy-folder/Cargo.toml index ba4a2e3481..c80a0bfdde 100644 --- a/frontend/rust-lib/flowy-folder/Cargo.toml +++ b/frontend/rust-lib/flowy-folder/Cargo.toml @@ -14,7 +14,7 @@ collab-plugins = { workspace = true } collab-integrate = { workspace = true } flowy-folder-pub = { workspace = true } flowy-search-pub = { workspace = true } - +flowy-sqlite = { workspace = true } flowy-derive.workspace = true flowy-notification = { workspace = true } parking_lot.workspace = true @@ -35,6 +35,7 @@ strum_macros = "0.21" protobuf.workspace = true uuid.workspace = true tokio-stream = { workspace = true, features = ["sync"] } +serde = { workspace = true, features = ["derive"]} serde_json.workspace = true validator = "0.16.0" async-trait.workspace = true diff --git a/frontend/rust-lib/flowy-folder/src/entities/workspace.rs b/frontend/rust-lib/flowy-folder/src/entities/workspace.rs index 21ff046226..e78eb6e307 100644 --- a/frontend/rust-lib/flowy-folder/src/entities/workspace.rs +++ b/frontend/rust-lib/flowy-folder/src/entities/workspace.rs @@ -232,3 +232,8 @@ pub struct UserFolderPB { #[pb(index = 2)] pub workspace_id: String, } + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Eq, PartialEq)] +pub struct IndexedWorkspaceIds { + pub workspace_ids: Vec, +} diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs index b3a8f6a1f3..71be3778ee 100644 --- a/frontend/rust-lib/flowy-folder/src/manager.rs +++ b/frontend/rust-lib/flowy-folder/src/manager.rs @@ -30,6 +30,7 @@ use flowy_error::{ErrorCode, FlowyError, FlowyResult}; use flowy_folder_pub::cloud::{gen_view_id, FolderCloudService}; use flowy_folder_pub::folder_builder::ParentChildViews; use flowy_search_pub::entities::FolderIndexManager; +use flowy_sqlite::kv::StorePreferences; use parking_lot::RwLock; use std::fmt::{Display, Formatter}; use std::ops::Deref; @@ -50,15 +51,17 @@ pub struct FolderManager { pub(crate) operation_handlers: FolderOperationHandlers, pub cloud_service: Arc, pub(crate) folder_indexer: Arc, + pub(crate) store_preferences: Arc, } impl FolderManager { - pub async fn new( + pub fn new( user: Arc, collab_builder: Arc, operation_handlers: FolderOperationHandlers, cloud_service: Arc, folder_indexer: Arc, + store_preferences: Arc, ) -> FlowyResult { let mutex_folder = Arc::new(MutexFolder::default()); let manager = Self { @@ -68,6 +71,7 @@ impl FolderManager { operation_handlers, cloud_service, folder_indexer, + store_preferences, }; Ok(manager) diff --git a/frontend/rust-lib/flowy-folder/src/manager_init.rs b/frontend/rust-lib/flowy-folder/src/manager_init.rs index 6332a683ff..07fc973f05 100644 --- a/frontend/rust-lib/flowy-folder/src/manager_init.rs +++ b/frontend/rust-lib/flowy-folder/src/manager_init.rs @@ -1,3 +1,4 @@ +use crate::entities::IndexedWorkspaceIds; use crate::manager::{FolderInitDataSource, FolderManager}; use crate::manager_observer::*; use crate::user_default::DefaultFolderBuilder; @@ -10,6 +11,8 @@ use std::sync::{Arc, Weak}; use tokio::task::spawn_blocking; use tracing::{event, info, Level}; +pub const INDEXED_WORKSPACE_KEY: &str = "indexed-workspace-ids"; + impl FolderManager { /// Called immediately after the application launched if the user already sign in/sign up. #[tracing::instrument(level = "info", skip(self, initial_data), err)] @@ -118,18 +121,7 @@ impl FolderManager { self .folder_indexer .set_index_content_receiver(index_content_rx, workspace_id.clone()); - - // Index all views in the folder if needed - if !self.folder_indexer.is_indexed() { - let views = folder.views.get_all_views(); - let folder_indexer = self.folder_indexer.clone(); - - // We spawn a blocking task to index all views in the folder - let wid = workspace_id.clone(); - spawn_blocking(move || { - folder_indexer.index_all_views(views, wid); - }); - } + self.handle_index_folder(workspace_id.clone(), &folder); *self.mutex_folder.write() = Some(folder); @@ -156,6 +148,7 @@ impl FolderManager { &weak_mutex_folder, Arc::downgrade(&self.user), ); + Ok(()) } @@ -192,4 +185,41 @@ impl FolderManager { folder_data, )) } + + fn handle_index_folder(&self, workspace_id: String, folder: &Folder) { + let index_all; + if self.folder_indexer.is_indexed() { + // If indexes already exist, we check if the workspace was + // previously indexed, if it wasn't we index all + let indexed_ids = self + .store_preferences + .get_object::(INDEXED_WORKSPACE_KEY); + if let Some(indexed_ids) = indexed_ids { + index_all = !indexed_ids.workspace_ids.contains(&workspace_id.clone()); + if !index_all { + let mut workspace_ids = indexed_ids.workspace_ids.clone(); + workspace_ids.push(workspace_id.clone()); + let _ = self + .store_preferences + .set_object(INDEXED_WORKSPACE_KEY, IndexedWorkspaceIds { workspace_ids }); + } + } else { + index_all = true; + } + } else { + // If there exists no indexes, we index all views in workspace + index_all = true; + } + + if index_all { + let views = folder.views.get_all_views(); + let folder_indexer = self.folder_indexer.clone(); + let wid = workspace_id.clone(); + + // We spawn a blocking task to index all views in the folder + spawn_blocking(move || { + folder_indexer.index_all_views(views, wid); + }); + } + } } diff --git a/frontend/rust-lib/flowy-search-pub/src/entities.rs b/frontend/rust-lib/flowy-search-pub/src/entities.rs index dfa741bf50..8b9b4a3cb6 100644 --- a/frontend/rust-lib/flowy-search-pub/src/entities.rs +++ b/frontend/rust-lib/flowy-search-pub/src/entities.rs @@ -13,6 +13,18 @@ pub struct IndexableData { pub workspace_id: String, } +impl IndexableData { + pub fn from_view(view: Arc, workspace_id: String) -> Self { + IndexableData { + id: view.id.clone(), + data: view.name.clone(), + icon: view.icon.clone(), + layout: view.layout.clone(), + workspace_id: workspace_id.clone(), + } + } +} + pub trait IndexManager: Send + Sync { fn set_index_content_receiver(&self, rx: IndexContentReceiver, workspace_id: String); fn add_index(&self, data: IndexableData) -> Result<(), FlowyError>; diff --git a/frontend/rust-lib/flowy-search/src/folder/indexer.rs b/frontend/rust-lib/flowy-search/src/folder/indexer.rs index 834efe949a..39192806b8 100644 --- a/frontend/rust-lib/flowy-search/src/folder/indexer.rs +++ b/frontend/rust-lib/flowy-search/src/folder/indexer.rs @@ -124,7 +124,7 @@ impl FolderIndexManagerImpl { } fn index_all(&self, indexes: Vec) -> Result<(), FlowyError> { - if self.is_indexed() || indexes.is_empty() { + if indexes.is_empty() { return Ok(()); } @@ -411,13 +411,7 @@ impl FolderIndexManager for FolderIndexManagerImpl { fn index_all_views(&self, views: Vec>, workspace_id: String) { let indexable_data = views .into_iter() - .map(|view| IndexableData { - id: view.id.clone(), - data: view.name.clone(), - icon: view.icon.clone(), - layout: view.layout.clone(), - workspace_id: workspace_id.clone(), - }) + .map(|view| IndexableData::from_view(view, workspace_id.clone())) .collect(); let _ = self.index_all(indexable_data);