From ff23165d3e9cd44a053390aaa7ef4cfc76720231 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:23:23 +0800 Subject: [PATCH] chore: download llm files (#5723) * chore: download file * chore: config download ui * chore: update zip * chore: config download ui * chore: unzip file * chore: unzip file * chore: rename * chore: disable local ai * chore: fmt * chore: fix warning * chore: update * chore: fix clippy --- .../shared/auth_operation.dart | 5 +- .../ai_chat/application/chat_bloc.dart | 3 +- .../ai_chat/application/chat_file_bloc.dart | 44 ++ .../lib/plugins/ai_chat/chat_page.dart | 51 +- .../settings/ai/download_model_bloc.dart | 140 +++++ .../settings/ai/local_ai_bloc.dart | 243 +++++++++ .../settings/ai/local_llm_listener.dart | 54 ++ .../settings/ai/plugin_state_bloc.dart | 36 ++ .../settings/ai/setting_local_ai_bloc.dart | 195 ------- .../pages/setting_ai_view/downloading.dart | 108 ++++ .../pages/setting_ai_view/init_local_ai.dart | 75 +++ .../setting_ai_view/local_ai_config.dart | 314 ++++++++++++ .../setting_ai_view/model_selection.dart | 85 +++ .../pages/setting_ai_view/plugin_state.dart | 45 ++ .../setting_ai_view/settings_ai_view.dart | 107 ++++ .../settings/pages/settings_ai_view.dart | 279 ---------- .../settings/settings_dialog.dart | 2 +- .../shared/af_dropdown_menu_entry.dart | 3 +- .../presentation/widgets/dialogs.dart | 4 +- frontend/appflowy_flutter/pubspec.lock | 8 + frontend/appflowy_flutter/pubspec.yaml | 1 + frontend/appflowy_tauri/src-tauri/Cargo.lock | 394 ++++++++++---- frontend/appflowy_tauri/src-tauri/Cargo.toml | 7 +- .../appflowy_web_app/src-tauri/Cargo.lock | 371 ++++++++++---- .../appflowy_web_app/src-tauri/Cargo.toml | 6 +- .../flowy_icons/16x/download_success.svg | 3 + .../flowy_icons/16x/download_warn.svg | 8 + .../flowy_icons/16x/local_model_download.svg | 5 + frontend/resources/translations/en.json | 10 +- frontend/rust-lib/Cargo.lock | 395 ++++++++++---- frontend/rust-lib/Cargo.toml | 10 +- .../event-integration-test/Cargo.toml | 2 +- .../event-integration-test/tests/util.rs | 2 +- frontend/rust-lib/flowy-chat-pub/src/cloud.rs | 13 +- frontend/rust-lib/flowy-chat/Cargo.toml | 10 +- frontend/rust-lib/flowy-chat/src/chat.rs | 34 +- .../rust-lib/flowy-chat/src/chat_manager.rs | 83 +-- frontend/rust-lib/flowy-chat/src/entities.rs | 149 +++++- .../rust-lib/flowy-chat/src/event_handler.rs | 105 +++- frontend/rust-lib/flowy-chat/src/event_map.rs | 46 +- frontend/rust-lib/flowy-chat/src/lib.rs | 1 + .../flowy-chat/src/local_ai/llm_resource.rs | 484 ++++++++++++++++++ .../flowy-chat/src/local_ai/local_llm_chat.rs | 254 +++++++++ .../rust-lib/flowy-chat/src/local_ai/mod.rs | 3 + .../flowy-chat/src/local_ai/model_request.rs | 149 ++++++ .../src/middleware/chat_service_mw.rs | 181 +++---- .../rust-lib/flowy-chat/src/notification.rs | 6 +- .../flowy-core/src/deps_resolve/chat_deps.rs | 5 + .../flowy-core/src/integrate/trait_impls.rs | 25 +- frontend/rust-lib/flowy-core/src/lib.rs | 12 +- frontend/rust-lib/flowy-error/Cargo.toml | 2 +- frontend/rust-lib/flowy-error/src/errors.rs | 8 +- frontend/rust-lib/flowy-search/Cargo.toml | 2 +- .../flowy-search/src/folder/indexer.rs | 8 +- .../flowy-search/tests/tantivy_test.rs | 4 +- .../flowy-server/src/af_cloud/impls/chat.rs | 35 +- .../af_cloud/impls/user/cloud_service_impl.rs | 10 +- .../rust-lib/flowy-server/src/default_impl.rs | 19 +- .../src/services/authenticate_user.rs | 5 + frontend/rust-lib/lib-infra/src/file_util.rs | 45 +- .../rust-lib/lib-infra/src/isolate_stream.rs | 5 +- 61 files changed, 3694 insertions(+), 1024 deletions(-) create mode 100644 frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_file_bloc.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/application/settings/ai/download_model_bloc.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_ai_bloc.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_llm_listener.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/application/settings/ai/plugin_state_bloc.dart delete mode 100644 frontend/appflowy_flutter/lib/workspace/application/settings/ai/setting_local_ai_bloc.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/downloading.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/init_local_ai.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/local_ai_config.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/model_selection.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/plugin_state.dart create mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/settings_ai_view.dart delete mode 100644 frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/settings_ai_view.dart create mode 100644 frontend/resources/flowy_icons/16x/download_success.svg create mode 100644 frontend/resources/flowy_icons/16x/download_warn.svg create mode 100644 frontend/resources/flowy_icons/16x/local_model_download.svg create mode 100644 frontend/rust-lib/flowy-chat/src/local_ai/llm_resource.rs create mode 100644 frontend/rust-lib/flowy-chat/src/local_ai/local_llm_chat.rs create mode 100644 frontend/rust-lib/flowy-chat/src/local_ai/mod.rs create mode 100644 frontend/rust-lib/flowy-chat/src/local_ai/model_request.rs diff --git a/frontend/appflowy_flutter/integration_test/shared/auth_operation.dart b/frontend/appflowy_flutter/integration_test/shared/auth_operation.dart index 9e205182a4..9c103ace86 100644 --- a/frontend/appflowy_flutter/integration_test/shared/auth_operation.dart +++ b/frontend/appflowy_flutter/integration_test/shared/auth_operation.dart @@ -13,7 +13,10 @@ import 'util.dart'; extension AppFlowyAuthTest on WidgetTester { Future tapGoogleLoginInButton() async { - await tapButton(find.byKey(const Key('signInWithGoogleButton'))); + await tapButton( + find.byKey(const Key('signInWithGoogleButton')), + milliseconds: 3000, + ); } /// Requires being on the SettingsPage.account of the SettingsDialog diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart index 8f3b5a14f8..2dc4926cc8 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart @@ -7,6 +7,7 @@ import 'package:appflowy_backend/dispatch/dispatch.dart'; import 'package:appflowy_backend/log.dart'; import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; import 'package:appflowy_backend/protobuf/flowy-error/code.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart'; import 'package:appflowy_backend/protobuf/flowy-folder/protobuf.dart'; import 'package:appflowy_backend/protobuf/flowy-folder/view.pb.dart'; import 'package:appflowy_backend/protobuf/flowy-user/user_profile.pb.dart'; @@ -480,7 +481,7 @@ class ChatState with _$ChatState { @freezed class LoadingState with _$LoadingState { const factory LoadingState.loading() = _Loading; - const factory LoadingState.finish() = _Finish; + const factory LoadingState.finish({FlowyError? error}) = _Finish; } enum OnetimeShotType { diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_file_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_file_bloc.dart new file mode 100644 index 0000000000..7f81dbcb53 --- /dev/null +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_file_bloc.dart @@ -0,0 +1,44 @@ +import 'package:appflowy_backend/dispatch/dispatch.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; +import 'package:freezed_annotation/freezed_annotation.dart'; + +part 'chat_file_bloc.freezed.dart'; + +class ChatFileBloc extends Bloc { + ChatFileBloc({ + required String chatId, + dynamic message, + }) : super(ChatFileState.initial(message)) { + on( + (event, emit) async { + await event.when( + initial: () async {}, + newFile: (String filePath) { + final payload = ChatFilePB(filePath: filePath, chatId: chatId); + ChatEventChatWithFile(payload).send(); + }, + ); + }, + ); + } +} + +@freezed +class ChatFileEvent with _$ChatFileEvent { + const factory ChatFileEvent.initial() = Initial; + const factory ChatFileEvent.newFile(String filePath) = _NewFile; +} + +@freezed +class ChatFileState with _$ChatFileState { + const factory ChatFileState({ + required String text, + }) = _ChatFileState; + + factory ChatFileState.initial(dynamic text) { + return ChatFileState( + text: text is String ? text : "", + ); + } +} diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart index c6a5e832c3..9c6a0eda0e 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart @@ -1,3 +1,5 @@ +import 'package:appflowy/plugins/ai_chat/application/chat_file_bloc.dart'; +import 'package:desktop_drop/desktop_drop.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; @@ -50,7 +52,7 @@ class AIChatUILayout { } } -class AIChatPage extends StatefulWidget { +class AIChatPage extends StatelessWidget { const AIChatPage({ super.key, required this.view, @@ -63,10 +65,53 @@ class AIChatPage extends StatefulWidget { final UserProfilePB userProfile; @override - State createState() => _AIChatPageState(); + Widget build(BuildContext context) { + if (userProfile.authenticator == AuthenticatorPB.AppFlowyCloud) { + return BlocProvider( + create: (context) => ChatFileBloc(chatId: view.id.toString()), + child: BlocBuilder( + builder: (context, state) { + return DropTarget( + onDragDone: (DropDoneDetails detail) async { + for (final file in detail.files) { + context + .read() + .add(ChatFileEvent.newFile(file.path)); + } + }, + child: _ChatContentPage( + view: view, + userProfile: userProfile, + ), + ); + }, + ), + ); + } + + return Center( + child: FlowyText( + LocaleKeys.chat_unsupportedCloudPrompt.tr(), + fontSize: 20, + ), + ); + } } -class _AIChatPageState extends State { +class _ChatContentPage extends StatefulWidget { + const _ChatContentPage({ + required this.view, + required this.userProfile, + }); + + final UserProfilePB userProfile; + final ViewPB view; + + @override + State<_ChatContentPage> createState() => _ChatContentPageState(); +} + +class _ChatContentPageState extends State<_ChatContentPage> { late types.User _user; @override diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/download_model_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/download_model_bloc.dart new file mode 100644 index 0000000000..782eb0e037 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/download_model_bloc.dart @@ -0,0 +1,140 @@ +import 'dart:async'; +import 'dart:ffi'; +import 'dart:isolate'; + +import 'package:appflowy/plugins/ai_chat/application/chat_bloc.dart'; +import 'package:appflowy_backend/dispatch/dispatch.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:bloc/bloc.dart'; +import 'package:freezed_annotation/freezed_annotation.dart'; +import 'package:fixnum/fixnum.dart'; +part 'download_model_bloc.freezed.dart'; + +class DownloadModelBloc extends Bloc { + DownloadModelBloc(LLMModelPB model) + : super(DownloadModelState(model: model)) { + on(_handleEvent); + } + + Future _handleEvent( + DownloadModelEvent event, + Emitter emit, + ) async { + await event.when( + started: () async { + final downloadStream = DownloadingStream(); + downloadStream.listen( + onModelPercentage: (name, percent) { + if (!isClosed) { + add( + DownloadModelEvent.updatePercent(name, percent), + ); + } + }, + onPluginPercentage: (percent) { + if (!isClosed) { + add(DownloadModelEvent.updatePercent("AppFlowy Plugin", percent)); + } + }, + onFinish: () { + add(const DownloadModelEvent.downloadFinish()); + }, + onError: (err) { + // emit(state.copyWith(downloadError: err)); + }, + ); + + final payload = + DownloadLLMPB(progressStream: Int64(downloadStream.nativePort)); + final result = await ChatEventDownloadLLMResource(payload).send(); + result.fold((_) { + emit( + state.copyWith( + downloadStream: downloadStream, + loadingState: const LoadingState.finish(), + downloadError: null, + ), + ); + }, (err) { + emit(state.copyWith(loadingState: LoadingState.finish(error: err))); + }); + }, + updatePercent: (String object, double percent) { + emit(state.copyWith(object: object, percent: percent)); + }, + downloadFinish: () { + emit(state.copyWith(isFinish: true)); + }, + ); + } +} + +@freezed +class DownloadModelEvent with _$DownloadModelEvent { + const factory DownloadModelEvent.started() = _Started; + const factory DownloadModelEvent.updatePercent( + String object, + double percent, + ) = _UpdatePercent; + const factory DownloadModelEvent.downloadFinish() = _DownloadFinish; +} + +@freezed +class DownloadModelState with _$DownloadModelState { + const factory DownloadModelState({ + required LLMModelPB model, + DownloadingStream? downloadStream, + String? downloadError, + @Default("") String object, + @Default(0) double percent, + @Default(false) bool isFinish, + @Default(LoadingState.loading()) LoadingState loadingState, + }) = _DownloadModelState; +} + +class DownloadingStream { + DownloadingStream() { + _port.handler = _controller.add; + } + + final RawReceivePort _port = RawReceivePort(); + StreamSubscription? _sub; + final StreamController _controller = StreamController.broadcast(); + int get nativePort => _port.sendPort.nativePort; + + Future dispose() async { + await _sub?.cancel(); + await _controller.close(); + _port.close(); + } + + void listen({ + void Function(String modelName, double percent)? onModelPercentage, + void Function(double percent)? onPluginPercentage, + void Function(String data)? onError, + void Function()? onFinish, + }) { + _sub = _controller.stream.listen((text) { + if (text.contains(':progress:')) { + final progressIndex = text.indexOf(':progress:'); + final modelName = text.substring(0, progressIndex); + final progressValue = text + .substring(progressIndex + 10); // 10 is the length of ":progress:" + final percent = double.tryParse(progressValue); + if (percent != null) { + onModelPercentage?.call(modelName, percent); + } + } else if (text.startsWith('plugin:progress:')) { + final percent = double.tryParse(text.substring(16)); + if (percent != null) { + onPluginPercentage?.call(percent); + } + } else if (text.startsWith('finish')) { + onFinish?.call(); + } else if (text.startsWith('error:')) { + // substring 6 to remove "error:" + onError?.call(text.substring(6)); + } + }); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_ai_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_ai_bloc.dart new file mode 100644 index 0000000000..15d91d6b96 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_ai_bloc.dart @@ -0,0 +1,243 @@ +import 'dart:async'; + +import 'package:appflowy/plugins/ai_chat/application/chat_bloc.dart'; +import 'package:appflowy/workspace/application/settings/ai/local_llm_listener.dart'; +import 'package:appflowy_backend/dispatch/dispatch.dart'; +import 'package:appflowy_backend/log.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart'; +import 'package:appflowy_result/appflowy_result.dart'; +import 'package:bloc/bloc.dart'; +import 'package:freezed_annotation/freezed_annotation.dart'; + +part 'local_ai_bloc.freezed.dart'; + +class LocalAISettingBloc + extends Bloc { + LocalAISettingBloc() + : listener = LocalLLMListener(), + super(const LocalAISettingState()) { + listener.start( + stateCallback: (newState) { + if (!isClosed) { + add(LocalAISettingEvent.updateLLMRunningState(newState)); + } + }, + ); + + on(_handleEvent); + } + + final LocalLLMListener listener; + + /// Handles incoming events and dispatches them to the appropriate handler. + Future _handleEvent( + LocalAISettingEvent event, + Emitter emit, + ) async { + await event.when( + started: _handleStarted, + didLoadModelInfo: (FlowyResult result) { + result.fold( + (modelInfo) { + _fetchCurremtLLMState(); + emit( + state.copyWith( + modelInfo: modelInfo, + models: modelInfo.models, + selectedLLMModel: modelInfo.selectedModel, + fetchModelInfoState: const LoadingState.finish(), + ), + ); + }, + (err) { + emit(state.copyWith(fetchModelInfoState: LoadingState.finish(error: err))); + }, + ); + }, + selectLLMConfig: (LLMModelPB llmModel) async { + final result = await ChatEventUpdateLocalLLM(llmModel).send(); + result.fold( + (llmResource) { + // If all resources are downloaded, show reload plugin + if (llmResource.pendingResources.isNotEmpty) { + emit( + state.copyWith( + selectedLLMModel: llmModel, + localAIInfo: LocalAIProgress.showDownload( + llmResource, + llmModel, + ), + selectLLMState: const LoadingState.finish(), + ), + ); + } else { + emit( + state.copyWith( + selectedLLMModel: llmModel, + selectLLMState: const LoadingState.finish(), + localAIInfo: const LocalAIProgress.checkPluginState(), + ), + ); + } + }, + (err) { + emit( + state.copyWith( + selectLLMState: LoadingState.finish(error: err), + ), + ); + }, + ); + }, + refreshLLMState: (LocalModelResourcePB llmResource) { + if (state.selectedLLMModel == null) { + Log.error( + 'Unexpected null selected config. It should be set already', + ); + return; + } + + // reload plugin if all resources are downloaded + if (llmResource.pendingResources.isEmpty) { + emit( + state.copyWith( + localAIInfo: const LocalAIProgress.checkPluginState(), + ), + ); + } else { + if (state.selectedLLMModel != null) { + // Go to download page if the selected model is downloading + if (llmResource.isDownloading) { + emit( + state.copyWith( + localAIInfo: + LocalAIProgress.startDownloading(state.selectedLLMModel!), + selectLLMState: const LoadingState.finish(), + ), + ); + return; + } else { + emit( + state.copyWith( + localAIInfo: LocalAIProgress.showDownload( + llmResource, + state.selectedLLMModel!, + ), + selectLLMState: const LoadingState.finish(), + ), + ); + } + } + } + }, + startDownloadModel: (LLMModelPB llmModel) { + emit( + state.copyWith( + localAIInfo: LocalAIProgress.startDownloading(llmModel), + selectLLMState: const LoadingState.finish(), + ), + ); + }, + cancelDownload: () async { + final _ = await ChatEventCancelDownloadLLMResource().send(); + _fetchCurremtLLMState(); + }, + finishDownload: () async { + emit( + state.copyWith(localAIInfo: const LocalAIProgress.finishDownload()), + ); + }, + updateLLMRunningState: (RunningStatePB newRunningState) { + if (newRunningState == RunningStatePB.Stopped) { + emit( + state.copyWith( + runningState: newRunningState, + localAIInfo: const LocalAIProgress.checkPluginState(), + ), + ); + } else { + emit(state.copyWith(runningState: newRunningState)); + } + }, + ); + } + + void _fetchCurremtLLMState() async { + final result = await ChatEventGetLocalLLMState().send(); + result.fold( + (llmResource) { + if (!isClosed) { + add(LocalAISettingEvent.refreshLLMState(llmResource)); + } + }, + (err) { + Log.error(err); + }, + ); + } + + /// Handles the event to fetch local AI settings when the application starts. + Future _handleStarted() async { + final result = await ChatEventRefreshLocalAIModelInfo().send(); + if (!isClosed) { + add(LocalAISettingEvent.didLoadModelInfo(result)); + } + } +} + +@freezed +class LocalAISettingEvent with _$LocalAISettingEvent { + const factory LocalAISettingEvent.started() = _Started; + const factory LocalAISettingEvent.didLoadModelInfo( + FlowyResult result, + ) = _ModelInfo; + const factory LocalAISettingEvent.selectLLMConfig(LLMModelPB config) = + _SelectLLMConfig; + + const factory LocalAISettingEvent.refreshLLMState( + LocalModelResourcePB llmResource, + ) = _RefreshLLMResource; + const factory LocalAISettingEvent.startDownloadModel(LLMModelPB llmModel) = + _StartDownloadModel; + + const factory LocalAISettingEvent.cancelDownload() = _CancelDownload; + const factory LocalAISettingEvent.finishDownload() = _FinishDownload; + const factory LocalAISettingEvent.updateLLMRunningState( + RunningStatePB newRunningState, + ) = _RunningState; +} + +@freezed +class LocalAISettingState with _$LocalAISettingState { + const factory LocalAISettingState({ + LLMModelInfoPB? modelInfo, + LLMModelPB? selectedLLMModel, + LocalAIProgress? localAIInfo, + @Default(LoadingState.loading()) LoadingState fetchModelInfoState, + @Default(LoadingState.loading()) LoadingState selectLLMState, + @Default([]) List models, + @Default(RunningStatePB.Connecting) RunningStatePB runningState, + }) = _LocalAISettingState; +} + +@freezed +class LocalAIProgress with _$LocalAIProgress { + // when user select a new model, it will call requestDownload + const factory LocalAIProgress.requestDownloadInfo( + LocalModelResourcePB llmResource, + LLMModelPB llmModel, + ) = _RequestDownload; + + // when user comes back to the setting page, it will auto detect current llm state + const factory LocalAIProgress.showDownload( + LocalModelResourcePB llmResource, + LLMModelPB llmModel, + ) = _DownloadNeeded; + + // when start downloading the model + const factory LocalAIProgress.startDownloading(LLMModelPB llmModel) = + _Downloading; + const factory LocalAIProgress.finishDownload() = _Finish; + const factory LocalAIProgress.checkPluginState() = _PluginState; +} diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_llm_listener.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_llm_listener.dart new file mode 100644 index 0000000000..f68d14abed --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/local_llm_listener.dart @@ -0,0 +1,54 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:appflowy/plugins/ai_chat/application/chat_notification.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/notification.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-notification/subject.pb.dart'; +import 'package:appflowy_backend/rust_stream.dart'; +import 'package:appflowy_result/appflowy_result.dart'; + +typedef PluginStateCallback = void Function(RunningStatePB state); + +class LocalLLMListener { + LocalLLMListener() { + _parser = + ChatNotificationParser(id: "appflowy_chat_plugin", callback: _callback); + _subscription = RustStreamReceiver.listen( + (observable) => _parser?.parse(observable), + ); + } + + StreamSubscription? _subscription; + ChatNotificationParser? _parser; + + PluginStateCallback? stateCallback; + void Function()? finishStreamingCallback; + + void start({ + PluginStateCallback? stateCallback, + }) { + this.stateCallback = stateCallback; + } + + void _callback( + ChatNotification ty, + FlowyResult result, + ) { + result.map((r) { + switch (ty) { + case ChatNotification.UpdateChatPluginState: + stateCallback?.call(PluginStatePB.fromBuffer(r).state); + break; + default: + break; + } + }); + } + + Future stop() async { + await _subscription?.cancel(); + _subscription = null; + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/plugin_state_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/plugin_state_bloc.dart new file mode 100644 index 0000000000..e4f61b8e99 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/plugin_state_bloc.dart @@ -0,0 +1,36 @@ +import 'dart:async'; + +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:bloc/bloc.dart'; +import 'package:freezed_annotation/freezed_annotation.dart'; +part 'plugin_state_bloc.freezed.dart'; + +class PluginStateBloc extends Bloc { + PluginStateBloc() + : super( + const PluginState(runningState: RunningStatePB.Connecting), + ) { + on(_handleEvent); + } + + Future _handleEvent( + PluginStateEvent event, + Emitter emit, + ) async { + await event.when( + started: () async {}, + ); + } +} + +@freezed +class PluginStateEvent with _$PluginStateEvent { + const factory PluginStateEvent.started() = _Started; +} + +@freezed +class PluginState with _$PluginState { + const factory PluginState({ + required RunningStatePB runningState, + }) = _PluginState; +} diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/setting_local_ai_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/ai/setting_local_ai_bloc.dart deleted file mode 100644 index e8d9c5eb31..0000000000 --- a/frontend/appflowy_flutter/lib/workspace/application/settings/ai/setting_local_ai_bloc.dart +++ /dev/null @@ -1,195 +0,0 @@ -import 'dart:io'; - -import 'package:appflowy/plugins/ai_chat/application/chat_bloc.dart'; -import 'package:appflowy_backend/dispatch/dispatch.dart'; -import 'package:appflowy_backend/log.dart'; -import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; -import 'package:bloc/bloc.dart'; -import 'package:freezed_annotation/freezed_annotation.dart'; -import 'package:path/path.dart' as path; -import 'package:protobuf/protobuf.dart'; - -part 'setting_local_ai_bloc.freezed.dart'; - -class SettingsAILocalBloc - extends Bloc { - SettingsAILocalBloc() : super(const SettingsAILocalState()) { - on(_handleEvent); - } - - /// Handles incoming events and dispatches them to the appropriate handler. - Future _handleEvent( - SettingsAILocalEvent event, - Emitter emit, - ) async { - await event.when( - started: _handleStarted, - didUpdateAISetting: (settings) async { - _handleDidUpdateAISetting(settings, emit); - }, - updateChatBin: (chatBinPath) async { - await _handleUpdatePath( - filePath: chatBinPath, - emit: emit, - stateUpdater: () => state.copyWith( - chatBinPath: chatBinPath.trim(), - chatBinPathError: null, - ), - errorUpdater: (error) => state.copyWith(chatBinPathError: error), - ); - }, - updateChatModelPath: (chatModelPath) async { - await _handleUpdatePath( - filePath: chatModelPath, - emit: emit, - stateUpdater: () => state.copyWith( - chatModelPath: chatModelPath.trim(), - chatModelPathError: null, - ), - errorUpdater: (error) => state.copyWith(chatModelPathError: error), - ); - }, - toggleLocalAI: () async { - emit(state.copyWith(localAIEnabled: !state.localAIEnabled)); - }, - saveSetting: () async { - _handleSaveSetting(); - }, - ); - } - - /// Handles the event to fetch local AI settings when the application starts. - Future _handleStarted() async { - final result = await ChatEventGetLocalAISetting().send(); - result.fold( - (setting) { - if (!isClosed) { - add(SettingsAILocalEvent.didUpdateAISetting(setting)); - } - }, - (err) => Log.error('Failed to get local AI setting: $err'), - ); - } - - /// Handles the event to update the AI settings in the state. - void _handleDidUpdateAISetting( - LocalLLMSettingPB settings, - Emitter emit, - ) { - final newState = state.copyWith( - aiSettings: settings, - chatBinPath: settings.chatBinPath, - chatModelPath: settings.chatModelPath, - localAIEnabled: settings.enabled, - loadingState: const LoadingState.finish(), - ); - emit(newState.copyWith(saveButtonEnabled: _saveButtonEnabled(newState))); - } - - /// Handles updating file paths (both chat binary and chat model paths). - Future _handleUpdatePath({ - required String filePath, - required Emitter emit, - required SettingsAILocalState Function() stateUpdater, - required SettingsAILocalState Function(String) errorUpdater, - }) async { - filePath = filePath.trim(); - if (filePath.isEmpty) { - emit(stateUpdater()); - return; - } - - final validationError = await _validatePath(filePath); - if (validationError != null) { - emit(errorUpdater(validationError)); - return; - } - - final newState = stateUpdater(); - emit(newState.copyWith(saveButtonEnabled: _saveButtonEnabled(newState))); - } - - /// Validates the provided file path. - Future _validatePath(String filePath) async { - if (!isAbsolutePath(filePath)) { - return "$filePath must be absolute"; - } - - if (!await pathExists(filePath)) { - return "$filePath does not exist"; - } - return null; - } - - /// Handles saving the updated settings. - void _handleSaveSetting() { - if (state.aiSettings == null) return; - state.aiSettings!.freeze(); - final newSetting = state.aiSettings!.rebuild((value) { - value - ..chatBinPath = state.chatBinPath ?? value.chatBinPath - ..chatModelPath = state.chatModelPath ?? value.chatModelPath - ..enabled = state.localAIEnabled; - }); - - ChatEventUpdateLocalAISetting(newSetting).send().then((result) { - result.fold( - (_) { - if (!isClosed) { - add(SettingsAILocalEvent.didUpdateAISetting(newSetting)); - } - }, - (err) => Log.error('Failed to update local AI setting: $err'), - ); - }); - } - - /// Determines if the save button should be enabled based on the state. - bool _saveButtonEnabled(SettingsAILocalState newState) { - return newState.chatBinPathError == null && - newState.chatModelPathError == null && - newState.chatBinPath != null && - newState.chatModelPath != null; - } -} - -@freezed -class SettingsAILocalEvent with _$SettingsAILocalEvent { - const factory SettingsAILocalEvent.started() = _Started; - const factory SettingsAILocalEvent.didUpdateAISetting( - LocalLLMSettingPB settings, - ) = _GetAISetting; - const factory SettingsAILocalEvent.updateChatBin(String chatBinPath) = - _UpdateChatBin; - const factory SettingsAILocalEvent.updateChatModelPath(String chatModelPath) = - _UpdateChatModelPath; - const factory SettingsAILocalEvent.toggleLocalAI() = _EnableLocalAI; - const factory SettingsAILocalEvent.saveSetting() = _SaveSetting; -} - -@freezed -class SettingsAILocalState with _$SettingsAILocalState { - const factory SettingsAILocalState({ - LocalLLMSettingPB? aiSettings, - String? chatBinPath, - String? chatBinPathError, - String? chatModelPath, - String? chatModelPathError, - @Default(false) bool localAIEnabled, - @Default(false) bool saveButtonEnabled, - @Default(LoadingState.loading()) LoadingState loadingState, - }) = _SettingsAILocalState; -} - -/// Checks if a given file path is absolute. -bool isAbsolutePath(String filePath) { - return path.isAbsolute(filePath); -} - -/// Checks if a given file or directory path exists. -Future pathExists(String filePath) async { - final file = File(filePath); - final directory = Directory(filePath); - - return await file.exists() || await directory.exists(); -} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/downloading.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/downloading.dart new file mode 100644 index 0000000000..08611542a4 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/downloading.dart @@ -0,0 +1,108 @@ +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/download_model_bloc.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flowy_infra/theme_extension.dart'; +import 'package:flowy_infra_ui/style_widget/button.dart'; +import 'package:flowy_infra_ui/style_widget/text.dart'; +import 'package:flowy_infra_ui/widget/spacing.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; +import 'package:percent_indicator/linear_percent_indicator.dart'; + +class DownloadingIndicator extends StatelessWidget { + const DownloadingIndicator({ + required this.llmModel, + required this.onCancel, + required this.onFinish, + super.key, + }); + final LLMModelPB llmModel; + final VoidCallback onCancel; + final VoidCallback onFinish; + + @override + Widget build(BuildContext context) { + return BlocProvider( + create: (context) => + DownloadModelBloc(llmModel)..add(const DownloadModelEvent.started()), + child: BlocListener( + listener: (context, state) { + if (state.isFinish) { + onFinish(); + } + }, + child: DecoratedBox( + decoration: BoxDecoration( + color: Theme.of(context).colorScheme.surfaceContainerHighest, + borderRadius: BorderRadius.circular(8), + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: Column( + children: [ + // const DownloadingPrompt(), + // const VSpace(12), + DownloadingProgressBar(onCancel: onCancel), + ], + ), + ), + ), + ), + ); + } +} + +class DownloadingProgressBar extends StatelessWidget { + const DownloadingProgressBar({required this.onCancel, super.key}); + + final VoidCallback onCancel; + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + FlowyText( + "${LocaleKeys.settings_aiPage_keys_downloadingModel.tr()}: ${state.object}", + fontSize: 11, + ), + IntrinsicHeight( + child: Row( + children: [ + Expanded( + child: LinearPercentIndicator( + lineHeight: 9.0, + percent: state.percent, + padding: EdgeInsets.zero, + progressColor: AFThemeExtension.of(context).success, + backgroundColor: + AFThemeExtension.of(context).progressBarBGColor, + barRadius: const Radius.circular(8), + trailing: FlowyText( + "${(state.percent * 100).toStringAsFixed(0)}%", + fontSize: 11, + color: AFThemeExtension.of(context).success, + ), + ), + ), + const HSpace(12), + FlowyButton( + useIntrinsicWidth: true, + text: FlowyText( + LocaleKeys.button_cancel.tr(), + fontSize: 11, + ), + onTap: onCancel, + ), + ], + ), + ), + ], + ); + }, + ); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/init_local_ai.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/init_local_ai.dart new file mode 100644 index 0000000000..7e0ca50092 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/init_local_ai.dart @@ -0,0 +1,75 @@ +import 'package:appflowy/generated/flowy_svgs.g.dart'; +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/local_ai_bloc.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flowy_infra_ui/style_widget/text.dart'; +import 'package:flowy_infra_ui/widget/spacing.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; + +class InitLocalAIIndicator extends StatelessWidget { + const InitLocalAIIndicator({super.key}); + + @override + Widget build(BuildContext context) { + return DecoratedBox( + decoration: const BoxDecoration( + color: Color(0xFFEDF7ED), + borderRadius: BorderRadius.all( + Radius.circular(4), + ), + ), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 6, vertical: 4), + child: BlocBuilder( + builder: (context, state) { + switch (state.runningState) { + case RunningStatePB.Connecting: + case RunningStatePB.Connected: + return Row( + children: [ + const HSpace(8), + FlowyText( + LocaleKeys.settings_aiPage_keys_localAILoading.tr(), + fontSize: 11, + color: const Color(0xFF1E4620), + ), + ], + ); + case RunningStatePB.Running: + return Row( + children: [ + const HSpace(8), + const FlowySvg( + FlowySvgs.download_success_s, + color: Color(0xFF2E7D32), + ), + const HSpace(6), + FlowyText( + LocaleKeys.settings_aiPage_keys_localAILoaded.tr(), + fontSize: 11, + color: const Color(0xFF1E4620), + ), + ], + ); + case RunningStatePB.Stopped: + return Row( + children: [ + const HSpace(8), + FlowyText( + LocaleKeys.settings_aiPage_keys_localAIStopped.tr(), + fontSize: 11, + color: const Color(0xFFC62828), + ), + ], + ); + default: + return const SizedBox.shrink(); + } + }, + ), + ), + ); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/local_ai_config.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/local_ai_config.dart new file mode 100644 index 0000000000..c011b379e6 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/local_ai_config.dart @@ -0,0 +1,314 @@ +import 'package:appflowy/generated/flowy_svgs.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/local_ai_bloc.dart'; +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/downloading.dart'; +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/init_local_ai.dart'; +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/plugin_state.dart'; +import 'package:appflowy/workspace/presentation/widgets/dialogs.dart'; +import 'package:appflowy_backend/protobuf/flowy-chat/entities.pb.dart'; +import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart'; +import 'package:flowy_infra_ui/flowy_infra_ui.dart'; +import 'package:flutter/material.dart'; + +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/settings_ai_bloc.dart'; +import 'package:appflowy/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart'; +import 'package:appflowy/workspace/presentation/settings/shared/settings_dropdown.dart'; +import 'package:appflowy_backend/protobuf/flowy-user/protobuf.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; + +class LocalModelConfig extends StatelessWidget { + const LocalModelConfig({super.key}); + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + if (state.aiSettings == null) { + return const SizedBox.shrink(); + } + + if (state.aiSettings!.aiModel != AIModelPB.LocalAIModel) { + return const SizedBox.shrink(); + } + + return BlocProvider( + create: (context) => + LocalAISettingBloc()..add(const LocalAISettingEvent.started()), + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 6), + child: Column( + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Flexible( + child: FlowyText.medium( + LocaleKeys.settings_aiPage_keys_llmModel.tr(), + fontSize: 14, + ), + ), + const Spacer(), + BlocBuilder( + builder: (context, state) { + return state.fetchModelInfoState.when( + loading: () => + const CircularProgressIndicator.adaptive(), + finish: (err) { + return (err == null) + ? const _SelectLocalModelDropdownMenu() + : const SizedBox.shrink(); + }, + ); + }, + ), + ], + ), + const IntrinsicHeight(child: _LocalLLMInfoWidget()), + ], + ), + ), + ); + }, + ); + } +} + +class _SelectLocalModelDropdownMenu extends StatelessWidget { + const _SelectLocalModelDropdownMenu(); + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + return Flexible( + child: SettingsDropdown( + key: const Key('_SelectLocalModelDropdownMenu'), + onChanged: (model) => context.read().add( + LocalAISettingEvent.selectLLMConfig(model), + ), + selectedOption: state.selectedLLMModel!, + options: state.models + .map( + (llm) => buildDropdownMenuEntry( + context, + value: llm, + label: llm.chatModel, + padding: const EdgeInsets.symmetric(vertical: 8), + ), + ) + .toList(), + ), + ); + }, + ); + } +} + +class _LocalLLMInfoWidget extends StatelessWidget { + const _LocalLLMInfoWidget(); + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + final error = errorFromState(state); + if (error == null) { + // If the error is null, handle selected llm model. + if (state.localAIInfo != null) { + final child = state.localAIInfo!.when( + requestDownloadInfo: ( + LocalModelResourcePB llmResource, + LLMModelPB llmModel, + ) { + _showDownloadDialog(context, llmResource, llmModel); + return const SizedBox.shrink(); + }, + showDownload: ( + LocalModelResourcePB llmResource, + LLMModelPB llmModel, + ) => + _ShowDownloadIndicator( + llmResource: llmResource, + llmModel: llmModel, + ), + startDownloading: (llmModel) { + return DownloadingIndicator( + key: UniqueKey(), + llmModel: llmModel, + onFinish: () => context + .read() + .add(const LocalAISettingEvent.finishDownload()), + onCancel: () => context + .read() + .add(const LocalAISettingEvent.cancelDownload()), + ); + }, + finishDownload: () => const InitLocalAIIndicator(), + checkPluginState: () => const CheckPluginStateIndicator(), + ); + + return Padding( + padding: const EdgeInsets.only(top: 14), + child: child, + ); + } else { + return const SizedBox.shrink(); + } + } else { + return FlowyText( + error.msg, + maxLines: 10, + ); + } + }, + ); + } + + void _showDownloadDialog( + BuildContext context, + LocalModelResourcePB llmResource, + LLMModelPB llmModel, + ) { + WidgetsBinding.instance.addPostFrameCallback( + (_) { + showDialog( + context: context, + barrierDismissible: false, + useRootNavigator: false, + builder: (dialogContext) { + return _LLMModelDownloadDialog( + llmResource: llmResource, + onOkPressed: () { + context.read().add( + LocalAISettingEvent.startDownloadModel( + llmModel, + ), + ); + }, + onCancelPressed: () { + context.read().add( + const LocalAISettingEvent.cancelDownload(), + ); + }, + ); + }, + ); + }, + debugLabel: 'localModel.download', + ); + } + + FlowyError? errorFromState(LocalAISettingState state) { + final err = state.fetchModelInfoState.when( + loading: () => null, + finish: (err) => err, + ); + + if (err == null) { + state.selectLLMState.when( + loading: () => null, + finish: (err) => err, + ); + } + + return err; + } +} + +class _LLMModelDownloadDialog extends StatelessWidget { + const _LLMModelDownloadDialog({ + required this.llmResource, + required this.onOkPressed, + required this.onCancelPressed, + }); + final LocalModelResourcePB llmResource; + final VoidCallback onOkPressed; + final VoidCallback onCancelPressed; + + @override + Widget build(BuildContext context) { + return NavigatorOkCancelDialog( + title: LocaleKeys.settings_aiPage_keys_downloadLLMPrompt.tr( + args: [ + llmResource.pendingResources[0].name, + ], + ), + message: llmResource.pendingResources[0].fileSize == 0 + ? "" + : LocaleKeys.settings_aiPage_keys_downloadLLMPromptDetail.tr( + args: [ + llmResource.pendingResources[0].name, + llmResource.pendingResources[0].fileSize.toString(), + ], + ), + okTitle: LocaleKeys.button_confirm.tr(), + cancelTitle: LocaleKeys.button_cancel.tr(), + onOkPressed: onOkPressed, + onCancelPressed: onCancelPressed, + titleUpperCase: false, + ); + } +} + +class _ShowDownloadIndicator extends StatelessWidget { + const _ShowDownloadIndicator({ + required this.llmResource, + required this.llmModel, + }); + final LocalModelResourcePB llmResource; + final LLMModelPB llmModel; + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + return Row( + children: [ + const Spacer(), + IntrinsicWidth( + child: SizedBox( + height: 30, + child: FlowyButton( + text: FlowyText( + LocaleKeys.settings_aiPage_keys_downloadAIModelButton.tr(), + fontSize: 14, + color: const Color(0xFF005483), + ), + leftIcon: const FlowySvg( + FlowySvgs.local_model_download_s, + color: Color(0xFF005483), + ), + onTap: () { + showDialog( + context: context, + barrierDismissible: false, + useRootNavigator: false, + builder: (dialogContext) { + return _LLMModelDownloadDialog( + llmResource: llmResource, + onOkPressed: () { + context.read().add( + LocalAISettingEvent.startDownloadModel( + llmModel, + ), + ); + }, + onCancelPressed: () { + context.read().add( + const LocalAISettingEvent.cancelDownload(), + ); + }, + ); + }, + ); + }, + ), + ), + ), + ], + ); + }, + ); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/model_selection.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/model_selection.dart new file mode 100644 index 0000000000..4a1115e8e1 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/model_selection.dart @@ -0,0 +1,85 @@ +import 'package:flutter/material.dart'; + +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/settings_ai_bloc.dart'; +import 'package:appflowy/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart'; +import 'package:appflowy/workspace/presentation/settings/shared/settings_dropdown.dart'; +import 'package:appflowy_backend/log.dart'; +import 'package:appflowy_backend/protobuf/flowy-user/protobuf.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flowy_infra_ui/style_widget/text.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; + +class AIModelSelection extends StatelessWidget { + const AIModelSelection({super.key}); + + @override + Widget build(BuildContext context) { + return BlocBuilder( + builder: (context, state) { + return Padding( + padding: const EdgeInsets.symmetric(vertical: 6), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Flexible( + child: FlowyText.medium( + LocaleKeys.settings_aiPage_keys_llmModelType.tr(), + fontSize: 14, + ), + ), + const Spacer(), + Flexible( + child: SettingsDropdown( + key: const Key('_AIModelSelection'), + onChanged: (model) => context + .read() + .add(SettingsAIEvent.selectModel(model)), + selectedOption: state.userProfile.aiModel, + options: _availableModels + .map( + (format) => buildDropdownMenuEntry( + context, + value: format, + label: _titleForAIModel(format), + ), + ) + .toList(), + ), + ), + ], + ), + ); + }, + ); + } +} + +List _availableModels = [ + AIModelPB.DefaultModel, + AIModelPB.Claude3Opus, + AIModelPB.Claude3Sonnet, + AIModelPB.GPT35, + AIModelPB.GPT4o, + // AIModelPB.LocalAIModel, +]; + +String _titleForAIModel(AIModelPB model) { + switch (model) { + case AIModelPB.DefaultModel: + return "Default"; + case AIModelPB.Claude3Opus: + return "Claude 3 Opus"; + case AIModelPB.Claude3Sonnet: + return "Claude 3 Sonnet"; + case AIModelPB.GPT35: + return "GPT-3.5"; + case AIModelPB.GPT4o: + return "GPT-4o"; + case AIModelPB.LocalAIModel: + return "Local"; + default: + Log.error("Unknown AI model: $model, fallback to default"); + return "Default"; + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/plugin_state.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/plugin_state.dart new file mode 100644 index 0000000000..66c000b892 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/plugin_state.dart @@ -0,0 +1,45 @@ +import 'package:appflowy/generated/flowy_svgs.g.dart'; +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/plugin_state_bloc.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flowy_infra_ui/style_widget/text.dart'; +import 'package:flowy_infra_ui/widget/spacing.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; + +class CheckPluginStateIndicator extends StatelessWidget { + const CheckPluginStateIndicator({super.key}); + + @override + Widget build(BuildContext context) { + return DecoratedBox( + decoration: const BoxDecoration( + color: Color(0xFFEDF7ED), + borderRadius: BorderRadius.all( + Radius.circular(4), + ), + ), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 6, vertical: 4), + child: BlocProvider( + create: (context) => PluginStateBloc(), + child: Row( + children: [ + const HSpace(8), + const FlowySvg( + FlowySvgs.download_success_s, + color: Color(0xFF2E7D32), + ), + const HSpace(6), + FlowyText( + LocaleKeys.settings_aiPage_keys_localAILoaded.tr(), + fontSize: 11, + color: const Color(0xFF1E4620), + ), + ], + ), + ), + ), + ); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/settings_ai_view.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/settings_ai_view.dart new file mode 100644 index 0000000000..e156bbd696 --- /dev/null +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/setting_ai_view/settings_ai_view.dart @@ -0,0 +1,107 @@ +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/local_ai_config.dart'; +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/model_selection.dart'; +import 'package:flutter/material.dart'; + +import 'package:appflowy/generated/locale_keys.g.dart'; +import 'package:appflowy/workspace/application/settings/ai/settings_ai_bloc.dart'; +import 'package:appflowy/workspace/presentation/settings/shared/settings_body.dart'; +import 'package:appflowy/workspace/presentation/widgets/toggle/toggle.dart'; +import 'package:appflowy_backend/protobuf/flowy-user/protobuf.dart'; +import 'package:easy_localization/easy_localization.dart'; +import 'package:flowy_infra_ui/style_widget/text.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; + +class AIFeatureOnlySupportedWhenUsingAppFlowyCloud extends StatelessWidget { + const AIFeatureOnlySupportedWhenUsingAppFlowyCloud({super.key}); + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.symmetric(horizontal: 60, vertical: 30), + child: FlowyText( + LocaleKeys.settings_aiPage_keys_loginToEnableAIFeature.tr(), + maxLines: null, + fontSize: 16, + lineHeight: 1.6, + ), + ); + } +} + +class SettingsAIView extends StatelessWidget { + const SettingsAIView({super.key, required this.userProfile}); + + final UserProfilePB userProfile; + + @override + Widget build(BuildContext context) { + return BlocProvider( + create: (_) => + SettingsAIBloc(userProfile)..add(const SettingsAIEvent.started()), + child: BlocBuilder( + builder: (context, state) { + final children = [ + const AIModelSelection(), + ]; + + if (state.aiSettings != null && + state.aiSettings!.aiModel == AIModelPB.LocalAIModel) { + children.add(const LocalModelConfig()); + } + + children.add(const _AISearchToggle(value: false)); + + return SettingsBody( + title: LocaleKeys.settings_aiPage_title.tr(), + description: + LocaleKeys.settings_aiPage_keys_aiSettingsDescription.tr(), + children: children, + ); + }, + ), + ); + } +} + +class _AISearchToggle extends StatelessWidget { + const _AISearchToggle({required this.value}); + + final bool value; + + @override + Widget build(BuildContext context) { + return Column( + children: [ + Row( + children: [ + FlowyText.medium( + LocaleKeys.settings_aiPage_keys_enableAISearchTitle.tr(), + ), + const Spacer(), + BlocBuilder( + builder: (context, state) { + if (state.aiSettings == null) { + return const Padding( + padding: EdgeInsets.only(top: 6), + child: SizedBox( + height: 26, + width: 26, + child: CircularProgressIndicator.adaptive(), + ), + ); + } else { + return Toggle( + value: state.enableSearchIndexing, + onChanged: (_) => context + .read() + .add(const SettingsAIEvent.toggleAISearch()), + ); + } + }, + ), + ], + ), + ], + ); + } +} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/settings_ai_view.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/settings_ai_view.dart deleted file mode 100644 index 1fba43b3dc..0000000000 --- a/frontend/appflowy_flutter/lib/workspace/presentation/settings/pages/settings_ai_view.dart +++ /dev/null @@ -1,279 +0,0 @@ -import 'package:appflowy/workspace/application/settings/ai/setting_local_ai_bloc.dart'; -import 'package:flowy_infra_ui/style_widget/button.dart'; -import 'package:flowy_infra_ui/widget/rounded_input_field.dart'; -import 'package:flowy_infra_ui/widget/spacing.dart'; -import 'package:flutter/material.dart'; - -import 'package:appflowy/generated/locale_keys.g.dart'; -import 'package:appflowy/workspace/application/settings/ai/settings_ai_bloc.dart'; -import 'package:appflowy/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart'; -import 'package:appflowy/workspace/presentation/settings/shared/settings_body.dart'; -import 'package:appflowy/workspace/presentation/settings/shared/settings_dropdown.dart'; -import 'package:appflowy/workspace/presentation/widgets/toggle/toggle.dart'; -import 'package:appflowy_backend/log.dart'; -import 'package:appflowy_backend/protobuf/flowy-user/protobuf.dart'; -import 'package:easy_localization/easy_localization.dart'; -import 'package:flowy_infra_ui/style_widget/text.dart'; -import 'package:flutter_bloc/flutter_bloc.dart'; - -class AIFeatureOnlySupportedWhenUsingAppFlowyCloud extends StatelessWidget { - const AIFeatureOnlySupportedWhenUsingAppFlowyCloud({super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.symmetric(horizontal: 60, vertical: 30), - child: FlowyText( - LocaleKeys.settings_aiPage_keys_loginToEnableAIFeature.tr(), - maxLines: null, - fontSize: 16, - lineHeight: 1.6, - ), - ); - } -} - -class SettingsAIView extends StatelessWidget { - const SettingsAIView({super.key, required this.userProfile}); - - final UserProfilePB userProfile; - - @override - Widget build(BuildContext context) { - return BlocProvider( - create: (_) => - SettingsAIBloc(userProfile)..add(const SettingsAIEvent.started()), - child: BlocBuilder( - builder: (context, state) { - return SettingsBody( - title: LocaleKeys.settings_aiPage_title.tr(), - description: - LocaleKeys.settings_aiPage_keys_aiSettingsDescription.tr(), - children: const [ - AIModelSelection(), - _AISearchToggle(value: false), - // Disable local AI configuration for now. It's not ready for production. - LocalAIConfiguration(), - ], - ); - }, - ), - ); - } -} - -class AIModelSelection extends StatelessWidget { - const AIModelSelection({super.key}); - - @override - Widget build(BuildContext context) { - return Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Flexible( - child: FlowyText.medium( - LocaleKeys.settings_aiPage_keys_llmModel.tr(), - ), - ), - const Spacer(), - Flexible( - child: BlocBuilder( - builder: (context, state) { - return SettingsDropdown( - key: const Key('AIModelDropdown'), - onChanged: (model) => context - .read() - .add(SettingsAIEvent.selectModel(model)), - selectedOption: state.userProfile.aiModel, - options: _availableModels - .map( - (format) => buildDropdownMenuEntry( - context, - value: format, - label: _titleForAIModel(format), - ), - ) - .toList(), - ); - }, - ), - ), - ], - ); - } -} - -List _availableModels = [ - AIModelPB.DefaultModel, - AIModelPB.Claude3Opus, - AIModelPB.Claude3Sonnet, - AIModelPB.GPT35, - AIModelPB.GPT4o, -]; - -String _titleForAIModel(AIModelPB model) { - switch (model) { - case AIModelPB.DefaultModel: - return "Default"; - case AIModelPB.Claude3Opus: - return "Claude 3 Opus"; - case AIModelPB.Claude3Sonnet: - return "Claude 3 Sonnet"; - case AIModelPB.GPT35: - return "GPT-3.5"; - case AIModelPB.GPT4o: - return "GPT-4o"; - case AIModelPB.LocalAIModel: - return "Local"; - default: - Log.error("Unknown AI model: $model, fallback to default"); - return "Default"; - } -} - -class _AISearchToggle extends StatelessWidget { - const _AISearchToggle({required this.value}); - - final bool value; - - @override - Widget build(BuildContext context) { - return Row( - children: [ - FlowyText.medium( - LocaleKeys.settings_aiPage_keys_enableAISearchTitle.tr(), - ), - const Spacer(), - BlocBuilder( - builder: (context, state) { - if (state.aiSettings == null) { - return const Padding( - padding: EdgeInsets.only(top: 6), - child: SizedBox( - height: 26, - width: 26, - child: CircularProgressIndicator.adaptive(), - ), - ); - } else { - return Toggle( - value: state.enableSearchIndexing, - onChanged: (_) => context - .read() - .add(const SettingsAIEvent.toggleAISearch()), - ); - } - }, - ), - ], - ); - } -} - -class LocalAIConfiguration extends StatelessWidget { - const LocalAIConfiguration({super.key}); - - @override - Widget build(BuildContext context) { - return BlocProvider( - create: (context) => - SettingsAILocalBloc()..add(const SettingsAILocalEvent.started()), - child: BlocBuilder( - builder: (context, state) { - return state.loadingState.when( - loading: () { - return const SizedBox.shrink(); - }, - finish: () { - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - AIConfigurateTextField( - title: 'chat bin path', - hitText: '', - errorText: state.chatBinPathError ?? '', - value: state.aiSettings?.chatBinPath ?? '', - onChanged: (value) { - context.read().add( - SettingsAILocalEvent.updateChatBin(value), - ); - }, - ), - const VSpace(16), - AIConfigurateTextField( - title: 'chat model path', - hitText: '', - errorText: state.chatModelPathError ?? '', - value: state.aiSettings?.chatModelPath ?? '', - onChanged: (value) { - context.read().add( - SettingsAILocalEvent.updateChatModelPath(value), - ); - }, - ), - const VSpace(16), - Toggle( - value: state.localAIEnabled, - onChanged: (_) => context - .read() - .add(const SettingsAILocalEvent.toggleLocalAI()), - ), - const VSpace(16), - FlowyButton( - disable: !state.saveButtonEnabled, - text: const FlowyText("save"), - onTap: () { - context.read().add( - const SettingsAILocalEvent.saveSetting(), - ); - }, - ), - ], - ); - }, - ); - }, - ), - ); - } -} - -class AIConfigurateTextField extends StatelessWidget { - const AIConfigurateTextField({ - required this.title, - required this.hitText, - required this.errorText, - required this.value, - required this.onChanged, - super.key, - }); - - final String title; - final String hitText; - final String errorText; - final String value; - final void Function(String) onChanged; - - @override - Widget build(BuildContext context) { - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - FlowyText( - title, - ), - const VSpace(8), - RoundedInputField( - hintText: hitText, - style: const TextStyle(fontSize: 14, fontWeight: FontWeight.w500), - normalBorderColor: Theme.of(context).colorScheme.outline, - errorBorderColor: Theme.of(context).colorScheme.error, - cursorColor: Theme.of(context).colorScheme.primary, - errorText: errorText, - initialValue: value, - onChanged: onChanged, - ), - ], - ); - } -} diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/settings_dialog.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/settings_dialog.dart index de7f167a45..9e79e3eb39 100644 --- a/frontend/appflowy_flutter/lib/workspace/presentation/settings/settings_dialog.dart +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/settings_dialog.dart @@ -4,7 +4,7 @@ import 'package:appflowy/startup/startup.dart'; import 'package:appflowy/workspace/application/settings/settings_dialog_bloc.dart'; import 'package:appflowy/workspace/application/user/user_workspace_bloc.dart'; import 'package:appflowy/workspace/presentation/settings/pages/settings_account_view.dart'; -import 'package:appflowy/workspace/presentation/settings/pages/settings_ai_view.dart'; +import 'package:appflowy/workspace/presentation/settings/pages/setting_ai_view/settings_ai_view.dart'; import 'package:appflowy/workspace/presentation/settings/pages/settings_billing_view.dart'; import 'package:appflowy/workspace/presentation/settings/pages/settings_manage_data_view.dart'; import 'package:appflowy/workspace/presentation/settings/pages/settings_plan_view.dart'; diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart b/frontend/appflowy_flutter/lib/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart index c58e3ecc26..919db8329b 100644 --- a/frontend/appflowy_flutter/lib/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart +++ b/frontend/appflowy_flutter/lib/workspace/presentation/settings/shared/af_dropdown_menu_entry.dart @@ -13,6 +13,7 @@ DropdownMenuEntry buildDropdownMenuEntry( Widget? leadingWidget, Widget? trailingWidget, String? fontFamily, + EdgeInsets padding = const EdgeInsets.symmetric(vertical: 4), }) { final fontFamilyUsed = fontFamily != null ? getGoogleFontSafely(fontFamily).fontFamily ?? defaultFontFamily @@ -32,7 +33,7 @@ DropdownMenuEntry buildDropdownMenuEntry( label: label, leadingIcon: leadingWidget, labelWidget: Padding( - padding: const EdgeInsets.symmetric(vertical: 4), + padding: padding, child: FlowyText.regular( label, fontSize: 14, diff --git a/frontend/appflowy_flutter/lib/workspace/presentation/widgets/dialogs.dart b/frontend/appflowy_flutter/lib/workspace/presentation/widgets/dialogs.dart index 1fe4cc7502..32949e54f8 100644 --- a/frontend/appflowy_flutter/lib/workspace/presentation/widgets/dialogs.dart +++ b/frontend/appflowy_flutter/lib/workspace/presentation/widgets/dialogs.dart @@ -184,6 +184,7 @@ class NavigatorOkCancelDialog extends StatelessWidget { this.title, this.message, this.maxWidth, + this.titleUpperCase = true, }); final VoidCallback? onOkPressed; @@ -193,6 +194,7 @@ class NavigatorOkCancelDialog extends StatelessWidget { final String? title; final String? message; final double? maxWidth; + final bool titleUpperCase; @override Widget build(BuildContext context) { @@ -204,7 +206,7 @@ class NavigatorOkCancelDialog extends StatelessWidget { children: [ if (title != null) ...[ FlowyText.medium( - title!.toUpperCase(), + titleUpperCase ? title!.toUpperCase() : title!, fontSize: FontSizes.s16, maxLines: 3, ), diff --git a/frontend/appflowy_flutter/pubspec.lock b/frontend/appflowy_flutter/pubspec.lock index 26ea4eca22..f330999d38 100644 --- a/frontend/appflowy_flutter/pubspec.lock +++ b/frontend/appflowy_flutter/pubspec.lock @@ -434,6 +434,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.10" + desktop_drop: + dependency: "direct main" + description: + name: desktop_drop + sha256: d55a010fe46c8e8fcff4ea4b451a9ff84a162217bdb3b2a0aa1479776205e15d + url: "https://pub.dev" + source: hosted + version: "0.4.4" device_info_plus: dependency: "direct main" description: diff --git a/frontend/appflowy_flutter/pubspec.yaml b/frontend/appflowy_flutter/pubspec.yaml index ea740c283c..b580961362 100644 --- a/frontend/appflowy_flutter/pubspec.yaml +++ b/frontend/appflowy_flutter/pubspec.yaml @@ -142,6 +142,7 @@ dependencies: shimmer: ^3.0.0 isolates: ^3.0.3+8 markdown_widget: ^2.3.2+6 + desktop_drop: ^0.4.4 markdown: # Window Manager for MacOS and Linux diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.lock b/frontend/appflowy_tauri/src-tauri/Cargo.lock index c4caa18f20..c77572fcac 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.lock +++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", @@ -172,7 +172,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -192,7 +192,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -206,22 +206,26 @@ dependencies = [ [[package]] name = "appflowy-local-ai" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "appflowy-plugin", "bytes", + "reqwest", "serde", "serde_json", "tokio", "tokio-stream", + "tokio-util", "tracing", + "zip 2.1.3", + "zip-extensions", ] [[package]] name = "appflowy-plugin" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "cfg-if", @@ -235,6 +239,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "xattr 1.3.1", ] [[package]] @@ -264,6 +269,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -378,6 +392,12 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -443,9 +463,9 @@ checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitpacking" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" dependencies = [ "crunchy", ] @@ -555,9 +575,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" @@ -806,7 +826,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "again", "anyhow", @@ -827,6 +847,7 @@ dependencies = [ "getrandom 0.2.10", "gotrue", "infra", + "lazy_static", "mime", "parking_lot 0.12.1", "percent-encoding", @@ -855,7 +876,7 @@ dependencies = [ [[package]] name = "client-api-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "collab-entity", "collab-rt-entity", @@ -867,7 +888,7 @@ dependencies = [ [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "futures-channel", "futures-util", @@ -1107,7 +1128,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -1132,7 +1153,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "async-trait", @@ -1200,6 +1221,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "convert_case" version = "0.4.0" @@ -1284,10 +1311,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.3.2" +name = "crc" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1486,7 +1528,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -1513,6 +1555,12 @@ dependencies = [ "regex", ] +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "delegate-display" version = "2.1.1" @@ -1525,6 +1573,16 @@ dependencies = [ "syn 2.0.47", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1536,6 +1594,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1660,6 +1729,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1858,9 +1938,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide 0.7.1", @@ -1883,6 +1963,7 @@ dependencies = [ "anyhow", "appflowy-local-ai", "appflowy-plugin", + "base64 0.21.5", "bytes", "dashmap", "flowy-chat-pub", @@ -1892,19 +1973,26 @@ dependencies = [ "flowy-notification", "flowy-sqlite", "futures", + "futures-util", "lib-dispatch", "lib-infra", "log", + "md5", "parking_lot 0.12.1", "protobuf", + "reqwest", "serde", "serde_json", + "sha2", "strum_macros 0.21.1", "tokio", "tokio-stream", + "tokio-util", "tracing", "uuid", "validator", + "zip 2.1.3", + "zip-extensions", ] [[package]] @@ -2518,12 +2606,12 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" dependencies = [ "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2940,7 +3028,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "futures-util", @@ -2957,7 +3045,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -3389,7 +3477,7 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -3451,9 +3539,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -3648,7 +3736,7 @@ dependencies = [ "tracing", "validator", "walkdir", - "zip", + "zip 0.6.6", ] [[package]] @@ -3751,6 +3839,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -3775,9 +3869,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.11.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ "hashbrown 0.14.3", ] @@ -3788,6 +3882,16 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mac" version = "0.1.1" @@ -3917,9 +4021,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.7.1" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -4125,6 +4229,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -4153,6 +4263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -4365,9 +4476,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" dependencies = [ "stable_deref_trait", ] @@ -4799,6 +4910,12 @@ dependencies = [ "reqwest", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4897,7 +5014,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4918,7 +5035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.47", @@ -5154,6 +5271,16 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -5856,9 +5983,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -5894,7 +6021,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=a2f92bb#a2f92bb13e7c3d33783359d02235ba4a5058a32f" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -5932,9 +6059,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" @@ -6263,14 +6390,13 @@ dependencies = [ [[package]] name = "tantivy" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6083cd777fa94271b8ce0fe4533772cb8110c3044bab048d20f70108329a1f2" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" dependencies = [ "aho-corasick 1.0.2", "arc-swap", - "async-trait", - "base64 0.21.5", + "base64 0.22.1", "bitpacking", "byteorder", "census", @@ -6278,16 +6404,16 @@ dependencies = [ "crossbeam-channel", "downcast-rs", "fastdivide", + "fnv", "fs4", "htmlescape", - "itertools 0.11.0", + "itertools 0.12.1", "levenshtein_automata", "log", "lru", "lz4_flex", "measure_time", "memmap2", - "murmurhash32", "num_cpus", "once_cell", "oneshot", @@ -6315,22 +6441,22 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" dependencies = [ + "downcast-rs", "fastdivide", - "fnv", - "itertools 0.11.0", + "itertools 0.12.1", "serde", "tantivy-bitpacker", "tantivy-common", @@ -6340,9 +6466,9 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" dependencies = [ "async-trait", "byteorder", @@ -6353,50 +6479,52 @@ dependencies = [ [[package]] name = "tantivy-fst" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.6.29", + "regex-syntax 0.8.4", "utf8-ranges", ] [[package]] name = "tantivy-query-grammar" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" dependencies = [ "nom", ] [[package]] name = "tantivy-sstable" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" dependencies = [ + "tantivy-bitpacker", "tantivy-common", "tantivy-fst", - "zstd 0.12.4", + "zstd 0.13.2", ] [[package]] name = "tantivy-stacker" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" dependencies = [ "murmurhash32", + "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" dependencies = [ "serde", ] @@ -6473,7 +6601,7 @@ checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" dependencies = [ "filetime", "libc", - "xattr", + "xattr 0.2.3", ] [[package]] @@ -6791,11 +6919,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa 1.0.6", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -6803,16 +6934,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -6927,16 +7059,19 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.14.3", "pin-project-lite", + "slab", "tokio", - "tracing", ] [[package]] @@ -8151,6 +8286,17 @@ dependencies = [ "libc", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "yrs" version = "0.19.1" @@ -8187,6 +8333,26 @@ dependencies = [ "syn 2.0.47", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "zip" version = "0.6.6" @@ -8196,7 +8362,7 @@ dependencies = [ "aes", "byteorder", "bzip2", - "constant_time_eq", + "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", "flate2", @@ -8207,6 +8373,58 @@ dependencies = [ "zstd 0.11.2+zstd.1.5.2", ] +[[package]] +name = "zip" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq 0.3.0", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap 2.1.0", + "lzma-rs", + "memchr", + "pbkdf2 0.12.2", + "rand 0.8.5", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd 0.13.2", +] + +[[package]] +name = "zip-extensions" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0a99499b3497d765525c5d05e3ade9ca4a731c184365c19472c3fd6ba86341" +dependencies = [ + "zip 2.1.3", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" @@ -8218,11 +8436,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 6.0.6", + "zstd-safe 7.2.0", ] [[package]] @@ -8237,21 +8455,19 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.toml b/frontend/appflowy_tauri/src-tauri/Cargo.toml index 759d4f64d2..8ba790b3a9 100644 --- a/frontend/appflowy_tauri/src-tauri/Cargo.toml +++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml @@ -29,6 +29,7 @@ tokio = "1.34.0" tokio-stream = "0.1.14" async-trait = "0.1.74" chrono = { version = "0.4.31", default-features = false, features = ["clock"] } +zip = "2.1.3" yrs = "0.19.1" # Please use the following script to update collab. # Working directory: frontend @@ -52,7 +53,7 @@ collab-user = { version = "0.2" } # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "a2f92bb" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "eebdbcad79a35b07305affdd36f16d9ce95c5a18" } [dependencies] serde_json.workspace = true @@ -127,5 +128,5 @@ collab-user = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy- # To update the commit ID, run: # scripts/tool/update_local_ai_rev.sh new_rev_id # ⚠️⚠️⚠️️ -appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } -appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } +appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } +appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } diff --git a/frontend/appflowy_web_app/src-tauri/Cargo.lock b/frontend/appflowy_web_app/src-tauri/Cargo.lock index 9ed5393c3f..6f2f64dcde 100644 --- a/frontend/appflowy_web_app/src-tauri/Cargo.lock +++ b/frontend/appflowy_web_app/src-tauri/Cargo.lock @@ -163,7 +163,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -183,7 +183,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -197,22 +197,26 @@ dependencies = [ [[package]] name = "appflowy-local-ai" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "appflowy-plugin", "bytes", + "reqwest", "serde", "serde_json", "tokio", "tokio-stream", + "tokio-util", "tracing", + "zip 2.1.3", + "zip-extensions", ] [[package]] name = "appflowy-plugin" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "cfg-if", @@ -226,6 +230,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "xattr", ] [[package]] @@ -254,6 +259,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arboard" version = "3.3.2" @@ -388,6 +402,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -453,9 +473,9 @@ checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitpacking" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" dependencies = [ "crunchy", ] @@ -544,9 +564,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" @@ -780,7 +800,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "again", "anyhow", @@ -801,6 +821,7 @@ dependencies = [ "getrandom 0.2.12", "gotrue", "infra", + "lazy_static", "mime", "parking_lot 0.12.1", "percent-encoding", @@ -829,7 +850,7 @@ dependencies = [ [[package]] name = "client-api-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "collab-entity", "collab-rt-entity", @@ -841,7 +862,7 @@ dependencies = [ [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "futures-channel", "futures-util", @@ -924,7 +945,7 @@ dependencies = [ [[package]] name = "collab" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "async-trait", @@ -948,7 +969,7 @@ dependencies = [ [[package]] name = "collab-database" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "async-trait", @@ -978,7 +999,7 @@ dependencies = [ [[package]] name = "collab-document" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "collab", @@ -998,7 +1019,7 @@ dependencies = [ [[package]] name = "collab-entity" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "bytes", @@ -1013,7 +1034,7 @@ dependencies = [ [[package]] name = "collab-folder" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "chrono", @@ -1051,7 +1072,7 @@ dependencies = [ [[package]] name = "collab-plugins" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "async-stream", @@ -1090,7 +1111,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -1115,7 +1136,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "async-trait", @@ -1132,7 +1153,7 @@ dependencies = [ [[package]] name = "collab-user" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=5048762#5048762dbb01abcbe75237e86c0d090e2f1d7c23" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=20f7814#20f7814beb265ea76e85ea7a9d392b9fe18a2a63" dependencies = [ "anyhow", "collab", @@ -1183,6 +1204,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "convert_case" version = "0.4.0" @@ -1280,10 +1307,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.4.0" +name = "crc" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1476,7 +1518,7 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -1503,6 +1545,12 @@ dependencies = [ "regex", ] +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "delegate-display" version = "2.1.1" @@ -1547,6 +1595,17 @@ dependencies = [ "syn 2.0.55", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1671,6 +1730,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "dlib" version = "0.5.2" @@ -1898,9 +1968,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1923,6 +1993,7 @@ dependencies = [ "anyhow", "appflowy-local-ai", "appflowy-plugin", + "base64 0.21.7", "bytes", "dashmap", "flowy-chat-pub", @@ -1932,19 +2003,26 @@ dependencies = [ "flowy-notification", "flowy-sqlite", "futures", + "futures-util", "lib-dispatch", "lib-infra", "log", + "md5", "parking_lot 0.12.1", "protobuf", + "reqwest", "serde", "serde_json", + "sha2", "strum_macros 0.21.1", "tokio", "tokio-stream", + "tokio-util", "tracing", "uuid", "validator", + "zip 2.1.3", + "zip-extensions", ] [[package]] @@ -2246,12 +2324,14 @@ dependencies = [ "flowy-notification", "flowy-search-pub", "flowy-sqlite", + "futures", "lazy_static", "lib-dispatch", "lib-infra", "nanoid", "parking_lot 0.12.1", "protobuf", + "regex", "serde", "serde_json", "strum_macros 0.21.1", @@ -2272,6 +2352,7 @@ dependencies = [ "collab-entity", "collab-folder", "lib-infra", + "serde", "uuid", ] @@ -2582,12 +2663,12 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" dependencies = [ "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3014,7 +3095,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "futures-util", @@ -3031,7 +3112,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -3468,7 +3549,7 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -3541,9 +3622,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -3732,7 +3813,7 @@ dependencies = [ "tracing", "validator", "walkdir", - "zip", + "zip 0.6.6", ] [[package]] @@ -3843,6 +3924,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -3867,9 +3954,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.11.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ "hashbrown 0.14.3", ] @@ -3880,6 +3967,16 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mac" version = "0.1.1" @@ -3989,15 +4086,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.7.1" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -4228,6 +4325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -4440,9 +4538,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" dependencies = [ "stable_deref_trait", ] @@ -4978,7 +5076,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4999,7 +5097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.55", @@ -5235,6 +5333,16 @@ dependencies = [ "getrandom 0.2.12", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -5977,7 +6085,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=9884d93aa2805013f36a79c1757174a0b5063065#9884d93aa2805013f36a79c1757174a0b5063065" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -6355,14 +6463,13 @@ dependencies = [ [[package]] name = "tantivy" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6083cd777fa94271b8ce0fe4533772cb8110c3044bab048d20f70108329a1f2" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" dependencies = [ "aho-corasick", "arc-swap", - "async-trait", - "base64 0.21.7", + "base64 0.22.1", "bitpacking", "byteorder", "census", @@ -6370,16 +6477,16 @@ dependencies = [ "crossbeam-channel", "downcast-rs", "fastdivide", + "fnv", "fs4", "htmlescape", - "itertools 0.11.0", + "itertools 0.12.1", "levenshtein_automata", "log", "lru", "lz4_flex", "measure_time", "memmap2", - "murmurhash32", "num_cpus", "once_cell", "oneshot", @@ -6407,22 +6514,22 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" dependencies = [ + "downcast-rs", "fastdivide", - "fnv", - "itertools 0.11.0", + "itertools 0.12.1", "serde", "tantivy-bitpacker", "tantivy-common", @@ -6432,9 +6539,9 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" dependencies = [ "async-trait", "byteorder", @@ -6445,50 +6552,52 @@ dependencies = [ [[package]] name = "tantivy-fst" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.6.29", + "regex-syntax 0.8.2", "utf8-ranges", ] [[package]] name = "tantivy-query-grammar" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" dependencies = [ "nom", ] [[package]] name = "tantivy-sstable" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" dependencies = [ + "tantivy-bitpacker", "tantivy-common", "tantivy-fst", - "zstd 0.12.4", + "zstd 0.13.2", ] [[package]] name = "tantivy-stacker" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" dependencies = [ "murmurhash32", + "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" dependencies = [ "serde", ] @@ -6852,18 +6961,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" dependencies = [ "proc-macro2", "quote", @@ -6904,9 +7013,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa 1.0.10", @@ -6925,9 +7034,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -7044,16 +7153,19 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.14.3", "pin-project-lite", + "slab", "tokio", - "tracing", ] [[package]] @@ -8445,9 +8557,9 @@ dependencies = [ [[package]] name = "yrs" -version = "0.18.8" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da227d69095141c331d9b60c11496d0a3c6505cd9f8e200898b197219e8e394f" +checksum = "71df0198938b69f1eec0d5f19f591c6e4f2f770b0bf16f858428f6d91b8bb280" dependencies = [ "arc-swap", "atomic_refcell", @@ -8479,6 +8591,26 @@ dependencies = [ "syn 2.0.55", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "zip" version = "0.6.6" @@ -8488,7 +8620,7 @@ dependencies = [ "aes", "byteorder", "bzip2", - "constant_time_eq", + "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", "flate2", @@ -8499,6 +8631,58 @@ dependencies = [ "zstd 0.11.2+zstd.1.5.2", ] +[[package]] +name = "zip" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq 0.3.0", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap 2.2.6", + "lzma-rs", + "memchr", + "pbkdf2 0.12.2", + "rand 0.8.5", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd 0.13.2", +] + +[[package]] +name = "zip-extensions" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0a99499b3497d765525c5d05e3ade9ca4a731c184365c19472c3fd6ba86341" +dependencies = [ + "zip 2.1.3", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" @@ -8510,11 +8694,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 6.0.6", + "zstd-safe 7.2.0", ] [[package]] @@ -8529,19 +8713,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/frontend/appflowy_web_app/src-tauri/Cargo.toml b/frontend/appflowy_web_app/src-tauri/Cargo.toml index d25584f6a6..e71778dcd4 100644 --- a/frontend/appflowy_web_app/src-tauri/Cargo.toml +++ b/frontend/appflowy_web_app/src-tauri/Cargo.toml @@ -52,7 +52,7 @@ collab-user = { version = "0.2" } # Run the script: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "a2f92bb" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "eebdbcad79a35b07305affdd36f16d9ce95c5a18" } [dependencies] serde_json.workspace = true @@ -128,6 +128,6 @@ collab-user = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy- # To update the commit ID, run: # scripts/tool/update_local_ai_rev.sh new_rev_id # ⚠️⚠️⚠️️ -appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } -appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } +appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } +appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } diff --git a/frontend/resources/flowy_icons/16x/download_success.svg b/frontend/resources/flowy_icons/16x/download_success.svg new file mode 100644 index 0000000000..d7cbd38d82 --- /dev/null +++ b/frontend/resources/flowy_icons/16x/download_success.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/resources/flowy_icons/16x/download_warn.svg b/frontend/resources/flowy_icons/16x/download_warn.svg new file mode 100644 index 0000000000..8f4f3810af --- /dev/null +++ b/frontend/resources/flowy_icons/16x/download_warn.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/frontend/resources/flowy_icons/16x/local_model_download.svg b/frontend/resources/flowy_icons/16x/local_model_download.svg new file mode 100644 index 0000000000..270bcb25d1 --- /dev/null +++ b/frontend/resources/flowy_icons/16x/local_model_download.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/resources/translations/en.json b/frontend/resources/translations/en.json index b2bceb1152..bc68914b9b 100644 --- a/frontend/resources/translations/en.json +++ b/frontend/resources/translations/en.json @@ -624,6 +624,14 @@ "aiSettingsDescription": "Select or configure Ai models used on @:appName. For best performance we recommend using the default model options", "loginToEnableAIFeature": "AI features are only enabled after logging in with @:appName Cloud. If you don't have an @:appName account, go to 'My Account' to sign up", "llmModel": "Language Model", + "llmModelType": "Language Model Type", + "downloadLLMPrompt": "Download {}", + "downloadLLMPromptDetail": "Downloading {} local model will take up to {} of storage. Do you want to continue?", + "downloadAIModelButton": "Download AI model", + "downloadingModel": "Downloading", + "localAILoaded": "Local AI Model successfully added and ready to use", + "localAILoading": "Local AI Model is loading...", + "localAIStopped": "Local AI Model stopped", "title": "AI API Keys", "openAILabel": "OpenAI API key", "openAITooltip": "You can find your Secret API key on the API key page", @@ -2080,4 +2088,4 @@ "signInError": "Sign in error", "login": "Sign up or log in" } -} +} \ No newline at end of file diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock index f726359968..fb17ab7072 100644 --- a/frontend/rust-lib/Cargo.lock +++ b/frontend/rust-lib/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", @@ -163,7 +163,7 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "app-error" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -183,7 +183,7 @@ dependencies = [ [[package]] name = "appflowy-ai-client" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -197,22 +197,26 @@ dependencies = [ [[package]] name = "appflowy-local-ai" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "appflowy-plugin", "bytes", + "reqwest", "serde", "serde_json", "tokio", "tokio-stream", + "tokio-util", "tracing", + "zip 2.1.3", + "zip-extensions", ] [[package]] name = "appflowy-plugin" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=0820a0d23f7b813dee505e7e29e88a8561699fe8#0820a0d23f7b813dee505e7e29e88a8561699fe8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-LocalAI?rev=b803d8a8fd6b1587449e869b3a7a1ac687cf630b#b803d8a8fd6b1587449e869b3a7a1ac687cf630b" dependencies = [ "anyhow", "cfg-if", @@ -226,6 +230,16 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "xattr", +] + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", ] [[package]] @@ -373,6 +387,12 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -438,9 +458,9 @@ checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitpacking" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" dependencies = [ "crunchy", ] @@ -544,9 +564,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" @@ -698,7 +718,7 @@ dependencies = [ [[package]] name = "client-api" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "again", "anyhow", @@ -748,7 +768,7 @@ dependencies = [ [[package]] name = "client-api-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "collab-entity", "collab-rt-entity", @@ -760,7 +780,7 @@ dependencies = [ [[package]] name = "client-websocket" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "futures-channel", "futures-util", @@ -969,7 +989,7 @@ dependencies = [ [[package]] name = "collab-rt-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bincode", @@ -994,7 +1014,7 @@ dependencies = [ [[package]] name = "collab-rt-protocol" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "async-trait", @@ -1083,6 +1103,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "cookie" version = "0.17.0" @@ -1137,10 +1163,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.3.2" +name = "crc" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1311,7 +1352,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "database-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -1338,6 +1379,12 @@ dependencies = [ "regex", ] +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "delegate-display" version = "2.1.1" @@ -1352,10 +1399,11 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -1370,6 +1418,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1450,6 +1509,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1588,7 +1658,7 @@ dependencies = [ "tracing", "uuid", "walkdir", - "zip", + "zip 2.1.3", ] [[package]] @@ -1679,9 +1749,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1704,6 +1774,7 @@ dependencies = [ "anyhow", "appflowy-local-ai", "appflowy-plugin", + "base64 0.21.5", "bytes", "dashmap", "dotenv", @@ -1714,21 +1785,28 @@ dependencies = [ "flowy-notification", "flowy-sqlite", "futures", + "futures-util", "lib-dispatch", "lib-infra", "log", + "md5", "parking_lot 0.12.1", "protobuf", + "reqwest", "serde", "serde_json", + "sha2", "simsimd", "strum_macros 0.21.1", "tokio", "tokio-stream", + "tokio-util", "tracing", "tracing-subscriber", "uuid", "validator", + "zip 2.1.3", + "zip-extensions", ] [[package]] @@ -2361,12 +2439,12 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" dependencies = [ "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2617,7 +2695,7 @@ dependencies = [ [[package]] name = "gotrue" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "futures-util", @@ -2634,7 +2712,7 @@ dependencies = [ [[package]] name = "gotrue-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -2999,7 +3077,7 @@ dependencies = [ [[package]] name = "infra" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "bytes", @@ -3048,9 +3126,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -3161,7 +3239,7 @@ dependencies = [ "tracing", "validator", "walkdir", - "zip", + "zip 0.6.6", ] [[package]] @@ -3254,6 +3332,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -3276,9 +3360,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.11.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ "hashbrown 0.14.3", ] @@ -3289,6 +3373,16 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mac" version = "0.1.1" @@ -3404,9 +3498,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.7.1" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -3568,6 +3662,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -3585,6 +3685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3699,9 +3800,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" dependencies = [ "stable_deref_trait", ] @@ -4094,6 +4195,12 @@ dependencies = [ "reqwest", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4182,7 +4289,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4203,7 +4310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.47", @@ -4480,6 +4587,16 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -4595,6 +4712,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -5062,9 +5185,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -5100,7 +5223,7 @@ dependencies = [ [[package]] name = "shared-entity" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=4d9108643b6b39b557f594a031f4fa20e9125ec8#4d9108643b6b39b557f594a031f4fa20e9125ec8" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=eebdbcad79a35b07305affdd36f16d9ce95c5a18#eebdbcad79a35b07305affdd36f16d9ce95c5a18" dependencies = [ "anyhow", "app-error", @@ -5136,6 +5259,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.4" @@ -5414,14 +5543,13 @@ dependencies = [ [[package]] name = "tantivy" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6083cd777fa94271b8ce0fe4533772cb8110c3044bab048d20f70108329a1f2" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" dependencies = [ "aho-corasick", "arc-swap", - "async-trait", - "base64 0.21.5", + "base64 0.22.1", "bitpacking", "byteorder", "census", @@ -5429,16 +5557,16 @@ dependencies = [ "crossbeam-channel", "downcast-rs", "fastdivide", + "fnv", "fs4", "htmlescape", - "itertools 0.11.0", + "itertools 0.12.1", "levenshtein_automata", "log", "lru", "lz4_flex", "measure_time", "memmap2", - "murmurhash32", "num_cpus", "once_cell", "oneshot", @@ -5466,22 +5594,22 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" dependencies = [ + "downcast-rs", "fastdivide", - "fnv", - "itertools 0.11.0", + "itertools 0.12.1", "serde", "tantivy-bitpacker", "tantivy-common", @@ -5491,9 +5619,9 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" dependencies = [ "async-trait", "byteorder", @@ -5504,50 +5632,52 @@ dependencies = [ [[package]] name = "tantivy-fst" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax 0.6.29", + "regex-syntax 0.8.4", "utf8-ranges", ] [[package]] name = "tantivy-query-grammar" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" dependencies = [ "nom", ] [[package]] name = "tantivy-sstable" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" dependencies = [ + "tantivy-bitpacker", "tantivy-common", "tantivy-fst", - "zstd 0.12.4", + "zstd 0.13.2", ] [[package]] name = "tantivy-stacker" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" dependencies = [ "murmurhash32", + "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" dependencies = [ "serde", ] @@ -5666,12 +5796,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -5679,16 +5811,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -5833,16 +5966,19 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.14.3", "pin-project-lite", + "slab", "tokio", - "tracing", ] [[package]] @@ -6728,6 +6864,17 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "yrs" version = "0.19.1" @@ -6764,6 +6911,26 @@ dependencies = [ "syn 2.0.47", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.47", +] + [[package]] name = "zip" version = "0.6.6" @@ -6773,7 +6940,7 @@ dependencies = [ "aes", "byteorder", "bzip2", - "constant_time_eq", + "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", "flate2", @@ -6784,6 +6951,58 @@ dependencies = [ "zstd 0.11.2+zstd.1.5.2", ] +[[package]] +name = "zip" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq 0.3.0", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap 2.1.0", + "lzma-rs", + "memchr", + "pbkdf2 0.12.2", + "rand 0.8.5", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd 0.13.2", +] + +[[package]] +name = "zip-extensions" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0a99499b3497d765525c5d05e3ade9ca4a731c184365c19472c3fd6ba86341" +dependencies = [ + "zip 2.1.3", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" @@ -6795,11 +7014,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 6.0.6", + "zstd-safe 7.2.0", ] [[package]] @@ -6814,21 +7033,19 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml index 2833e1e7f7..ad3c11761d 100644 --- a/frontend/rust-lib/Cargo.toml +++ b/frontend/rust-lib/Cargo.toml @@ -91,14 +91,16 @@ collab-plugins = { version = "0.2" } collab-user = { version = "0.2" } yrs = "0.19.1" validator = { version = "0.16.1", features = ["derive"] } +tokio-util = "0.7.11" +zip = "2.1.3" # Please using the following command to update the revision id # Current directory: frontend # Run the script.add_workspace_members: # scripts/tool/update_client_api_rev.sh new_rev_id # ⚠️⚠️⚠️️ -client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "4d9108643b6b39b557f594a031f4fa20e9125ec8" } -client-api-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "4d9108643b6b39b557f594a031f4fa20e9125ec8" } +client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "eebdbcad79a35b07305affdd36f16d9ce95c5a18" } +client-api-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "eebdbcad79a35b07305affdd36f16d9ce95c5a18" } [profile.dev] opt-level = 1 @@ -149,5 +151,5 @@ collab-user = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy- # To update the commit ID, run: # scripts/tool/update_local_ai_rev.sh new_rev_id # ⚠️⚠️⚠️️ -appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } -appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "0820a0d23f7b813dee505e7e29e88a8561699fe8" } +appflowy-local-ai = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } +appflowy-plugin = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-LocalAI", rev = "b803d8a8fd6b1587449e869b3a7a1ac687cf630b" } diff --git a/frontend/rust-lib/event-integration-test/Cargo.toml b/frontend/rust-lib/event-integration-test/Cargo.toml index a644709791..b836e8a40c 100644 --- a/frontend/rust-lib/event-integration-test/Cargo.toml +++ b/frontend/rust-lib/event-integration-test/Cargo.toml @@ -55,7 +55,7 @@ uuid.workspace = true assert-json-diff = "2.0.2" tokio-postgres = { version = "0.7.8" } chrono = "0.4.31" -zip = "0.6.6" +zip.workspace = true walkdir = "2.5.0" futures = "0.3.30" flowy-chat-pub = { workspace = true } diff --git a/frontend/rust-lib/event-integration-test/tests/util.rs b/frontend/rust-lib/event-integration-test/tests/util.rs index 0a63ccaac7..ad1a01bcff 100644 --- a/frontend/rust-lib/event-integration-test/tests/util.rs +++ b/frontend/rust-lib/event-integration-test/tests/util.rs @@ -190,7 +190,7 @@ pub fn zip(src_dir: PathBuf, output_file_path: PathBuf) -> io::Result<()> { .truncate(true) .open(&output_file_path)?; - let options = FileOptions::default().compression_method(CompressionMethod::Deflated); + let options = FileOptions::<()>::default().compression_method(CompressionMethod::Deflated); let mut zip = ZipWriter::new(file); diff --git a/frontend/rust-lib/flowy-chat-pub/src/cloud.rs b/frontend/rust-lib/flowy-chat-pub/src/cloud.rs index fae11d979e..0054891140 100644 --- a/frontend/rust-lib/flowy-chat-pub/src/cloud.rs +++ b/frontend/rust-lib/flowy-chat-pub/src/cloud.rs @@ -1,6 +1,7 @@ use bytes::Bytes; pub use client_api::entity::ai_dto::{ - CompletionType, RelatedQuestion, RepeatedRelatedQuestion, StringOrMessage, + AppFlowyAIPlugin, CompletionType, LLMModel, LocalAIConfig, ModelInfo, RelatedQuestion, + RepeatedRelatedQuestion, StringOrMessage, }; pub use client_api::entity::{ ChatAuthorType, ChatMessage, ChatMessageType, MessageCursor, QAChatMessage, RepeatedChatMessage, @@ -10,6 +11,7 @@ use flowy_error::FlowyError; use futures::stream::BoxStream; use lib_infra::async_trait::async_trait; use lib_infra::future::FutureResult; +use std::path::PathBuf; pub type ChatMessageStream = BoxStream<'static, Result>; pub type StreamAnswer = BoxStream<'static, Result>; @@ -74,4 +76,13 @@ pub trait ChatCloudService: Send + Sync + 'static { text: &str, complete_type: CompletionType, ) -> Result; + + async fn index_file( + &self, + workspace_id: &str, + file_path: PathBuf, + chat_id: &str, + ) -> Result<(), FlowyError>; + + async fn get_local_ai_config(&self, workspace_id: &str) -> Result; } diff --git a/frontend/rust-lib/flowy-chat/Cargo.toml b/frontend/rust-lib/flowy-chat/Cargo.toml index a782708330..91bee1a7ff 100644 --- a/frontend/rust-lib/flowy-chat/Cargo.toml +++ b/frontend/rust-lib/flowy-chat/Cargo.toml @@ -32,9 +32,17 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } anyhow = "1.0.86" tokio-stream = "0.1.15" +tokio-util = { workspace = true, features = ["full"] } parking_lot.workspace = true appflowy-local-ai = { version = "0.1.0", features = ["verbose"] } -appflowy-plugin = { version = "0.1.0", features = ["verbose"] } +appflowy-plugin = { version = "0.1.0" } +reqwest = "0.11.27" +sha2 = "0.10.7" +base64 = "0.21.5" +futures-util = "0.3.30" +md5 = "0.7.0" +zip = { workspace = true, features = ["deflate"] } +zip-extensions = "0.8.0" [dev-dependencies] dotenv = "0.15.0" diff --git a/frontend/rust-lib/flowy-chat/src/chat.rs b/frontend/rust-lib/flowy-chat/src/chat.rs index def485326e..6f3cff2a8d 100644 --- a/frontend/rust-lib/flowy-chat/src/chat.rs +++ b/frontend/rust-lib/flowy-chat/src/chat.rs @@ -2,7 +2,7 @@ use crate::chat_manager::ChatUserService; use crate::entities::{ ChatMessageErrorPB, ChatMessageListPB, ChatMessagePB, RepeatedRelatedQuestionPB, }; -use crate::middleware::chat_service_mw::ChatService; +use crate::middleware::chat_service_mw::ChatServiceMiddleware; use crate::notification::{send_notification, ChatNotification}; use crate::persistence::{insert_chat_messages, select_chat_messages, ChatMessageTable}; use allo_isolate::Isolate; @@ -11,6 +11,7 @@ use flowy_error::{FlowyError, FlowyResult}; use flowy_sqlite::DBConnection; use futures::{SinkExt, StreamExt}; use lib_infra::isolate_stream::IsolateSink; +use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicI64}; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; @@ -26,7 +27,7 @@ pub struct Chat { chat_id: String, uid: i64, user_service: Arc, - chat_service: Arc, + chat_service: Arc, prev_message_state: Arc>, latest_message_id: Arc, stop_stream: Arc, @@ -38,7 +39,7 @@ impl Chat { uid: i64, chat_id: String, user_service: Arc, - chat_service: Arc, + chat_service: Arc, ) -> Chat { Chat { uid, @@ -435,6 +436,33 @@ impl Chat { Ok(messages) } + + #[instrument(level = "debug", skip_all, err)] + pub async fn index_file(&self, file_path: PathBuf) -> FlowyResult<()> { + if !file_path.exists() { + return Err( + FlowyError::record_not_found().with_context(format!("{:?} not exist", file_path)), + ); + } + + if !file_path.is_file() { + return Err( + FlowyError::invalid_data().with_context(format!("{:?} is not a file ", file_path)), + ); + } + + trace!( + "[Chat] index file: chat_id={}, file_path={:?}", + self.chat_id, + file_path + ); + self + .chat_service + .index_file(&self.user_service.workspace_id()?, file_path, &self.chat_id) + .await?; + + Ok(()) + } } fn save_chat_message( diff --git a/frontend/rust-lib/flowy-chat/src/chat_manager.rs b/frontend/rust-lib/flowy-chat/src/chat_manager.rs index b900eb3e5c..b504a93104 100644 --- a/frontend/rust-lib/flowy-chat/src/chat_manager.rs +++ b/frontend/rust-lib/flowy-chat/src/chat_manager.rs @@ -1,33 +1,36 @@ use crate::chat::Chat; use crate::entities::{ChatMessageListPB, ChatMessagePB, RepeatedRelatedQuestionPB}; -use crate::middleware::chat_service_mw::ChatService; +use crate::local_ai::local_llm_chat::LocalAIController; +use crate::middleware::chat_service_mw::ChatServiceMiddleware; use crate::persistence::{insert_chat, ChatTable}; -use appflowy_local_ai::llm_chat::{LocalChatLLMChat, LocalLLMSetting}; + use appflowy_plugin::manager::PluginManager; use dashmap::DashMap; use flowy_chat_pub::cloud::{ChatCloudService, ChatMessageType}; use flowy_error::{FlowyError, FlowyResult}; use flowy_sqlite::kv::KVStorePreferences; use flowy_sqlite::DBConnection; + use lib_infra::util::timestamp; +use std::path::PathBuf; use std::sync::Arc; -use tracing::trace; +use tracing::{error, info, trace}; pub trait ChatUserService: Send + Sync + 'static { fn user_id(&self) -> Result; fn device_id(&self) -> Result; fn workspace_id(&self) -> Result; fn sqlite_connection(&self, uid: i64) -> Result; + fn user_data_dir(&self) -> Result; } pub struct ChatManager { - pub chat_service: Arc, + pub chat_service_wm: Arc, pub user_service: Arc, chats: Arc>>, - store_preferences: Arc, + pub local_ai_controller: Arc, } -const LOCAL_AI_SETTING_KEY: &str = "local_ai_setting"; impl ChatManager { pub fn new( cloud_service: Arc, @@ -35,42 +38,35 @@ impl ChatManager { store_preferences: Arc, ) -> ChatManager { let user_service = Arc::new(user_service); - let local_ai_setting = store_preferences - .get_object::(LOCAL_AI_SETTING_KEY) - .unwrap_or_default(); let plugin_manager = Arc::new(PluginManager::new()); + let local_ai_controller = Arc::new(LocalAIController::new( + plugin_manager.clone(), + store_preferences.clone(), + user_service.clone(), + cloud_service.clone(), + )); + + if local_ai_controller.is_ready() { + if let Err(err) = local_ai_controller.initialize() { + error!("[AI Plugin] failed to initialize local ai: {:?}", err); + } + } - // setup local AI chat plugin - let local_llm_ctrl = Arc::new(LocalChatLLMChat::new(plugin_manager)); // setup local chat service - let chat_service = Arc::new(ChatService::new( + let chat_service_wm = Arc::new(ChatServiceMiddleware::new( user_service.clone(), cloud_service, - local_llm_ctrl, - local_ai_setting, + local_ai_controller.clone(), )); Self { - chat_service, + chat_service_wm, user_service, chats: Arc::new(DashMap::new()), - store_preferences, + local_ai_controller, } } - pub fn update_local_ai_setting(&self, setting: LocalLLMSetting) -> FlowyResult<()> { - self.chat_service.update_local_ai_setting(setting.clone())?; - self - .store_preferences - .set_object(LOCAL_AI_SETTING_KEY, setting)?; - Ok(()) - } - - pub fn get_local_ai_setting(&self) -> FlowyResult { - let setting = self.chat_service.get_local_ai_setting(); - Ok(setting) - } - pub async fn open_chat(&self, chat_id: &str) -> Result<(), FlowyError> { trace!("open chat: {}", chat_id); self.chats.entry(chat_id.to_string()).or_insert_with(|| { @@ -78,24 +74,33 @@ impl ChatManager { self.user_service.user_id().unwrap(), chat_id.to_string(), self.user_service.clone(), - self.chat_service.clone(), + self.chat_service_wm.clone(), )) }); - self.chat_service.notify_open_chat(chat_id); + trace!("[AI Plugin] notify open chat: {}", chat_id); + self.local_ai_controller.open_chat(chat_id); Ok(()) } pub async fn close_chat(&self, chat_id: &str) -> Result<(), FlowyError> { trace!("close chat: {}", chat_id); - self.chat_service.notify_close_chat(chat_id); + + if self.local_ai_controller.is_ready() { + info!("[AI Plugin] notify close chat: {}", chat_id); + self.local_ai_controller.close_chat(chat_id); + } Ok(()) } pub async fn delete_chat(&self, chat_id: &str) -> Result<(), FlowyError> { if let Some((_, chat)) = self.chats.remove(chat_id) { chat.close(); - self.chat_service.notify_close_chat(chat_id); + + if self.local_ai_controller.is_ready() { + info!("[AI Plugin] notify close chat: {}", chat_id); + self.local_ai_controller.close_chat(chat_id); + } } Ok(()) } @@ -103,7 +108,7 @@ impl ChatManager { pub async fn create_chat(&self, uid: &i64, chat_id: &str) -> Result, FlowyError> { let workspace_id = self.user_service.workspace_id()?; self - .chat_service + .chat_service_wm .create_chat(uid, &workspace_id, chat_id) .await?; save_chat(self.user_service.sqlite_connection(*uid)?, chat_id)?; @@ -112,7 +117,7 @@ impl ChatManager { self.user_service.user_id().unwrap(), chat_id.to_string(), self.user_service.clone(), - self.chat_service.clone(), + self.chat_service_wm.clone(), )); self.chats.insert(chat_id.to_string(), chat.clone()); Ok(chat) @@ -140,7 +145,7 @@ impl ChatManager { self.user_service.user_id().unwrap(), chat_id.to_string(), self.user_service.clone(), - self.chat_service.clone(), + self.chat_service_wm.clone(), )); self.chats.insert(chat_id.to_string(), chat.clone()); Ok(chat) @@ -215,6 +220,12 @@ impl ChatManager { chat.stop_stream_message().await; Ok(()) } + + pub async fn chat_with_file(&self, chat_id: &str, file_path: PathBuf) -> FlowyResult<()> { + let chat = self.get_or_create_chat_instance(chat_id).await?; + chat.index_file(file_path).await?; + Ok(()) + } } fn save_chat(conn: DBConnection, chat_id: &str) -> FlowyResult<()> { diff --git a/frontend/rust-lib/flowy-chat/src/entities.rs b/frontend/rust-lib/flowy-chat/src/entities.rs index da4dcba029..67b93c8f64 100644 --- a/frontend/rust-lib/flowy-chat/src/entities.rs +++ b/frontend/rust-lib/flowy-chat/src/entities.rs @@ -1,6 +1,8 @@ -use appflowy_local_ai::llm_chat::LocalLLMSetting; +use crate::local_ai::local_llm_chat::LLMModelInfo; +use appflowy_plugin::core::plugin::RunningState; + use flowy_chat_pub::cloud::{ - ChatMessage, RelatedQuestion, RepeatedChatMessage, RepeatedRelatedQuestion, + ChatMessage, LLMModel, RelatedQuestion, RepeatedChatMessage, RepeatedRelatedQuestion, }; use flowy_derive::{ProtoBuf, ProtoBuf_Enum}; use lib_infra::validator_fn::required_not_empty_str; @@ -208,33 +210,49 @@ impl From for RepeatedRelatedQuestionPB { } #[derive(Debug, Clone, Default, ProtoBuf)] -pub struct LocalLLMSettingPB { +pub struct LLMModelInfoPB { #[pb(index = 1)] - pub chat_bin_path: String, + pub selected_model: LLMModelPB, #[pb(index = 2)] - pub chat_model_path: String, - - #[pb(index = 3)] - pub enabled: bool, + pub models: Vec, } -impl From for LocalLLMSettingPB { - fn from(value: LocalLLMSetting) -> Self { - LocalLLMSettingPB { - chat_bin_path: value.chat_bin_path, - chat_model_path: value.chat_model_path, - enabled: value.enabled, +impl From for LLMModelInfoPB { + fn from(value: LLMModelInfo) -> Self { + LLMModelInfoPB { + selected_model: LLMModelPB::from(value.selected_model), + models: value.models.into_iter().map(LLMModelPB::from).collect(), } } } -impl From for LocalLLMSetting { - fn from(value: LocalLLMSettingPB) -> Self { - LocalLLMSetting { - chat_bin_path: value.chat_bin_path, - chat_model_path: value.chat_model_path, - enabled: value.enabled, +#[derive(Debug, Clone, Default, ProtoBuf)] +pub struct LLMModelPB { + #[pb(index = 1)] + pub llm_id: i64, + + #[pb(index = 2)] + pub embedding_model: String, + + #[pb(index = 3)] + pub chat_model: String, + + #[pb(index = 4)] + pub requirement: String, + + #[pb(index = 5)] + pub file_size: i64, +} + +impl From for LLMModelPB { + fn from(value: LLMModel) -> Self { + LLMModelPB { + llm_id: value.llm_id, + embedding_model: value.embedding_model.name, + chat_model: value.chat_model.name, + requirement: value.chat_model.requirements, + file_size: value.chat_model.file_size, } } } @@ -283,3 +301,94 @@ pub enum ModelTypePB { #[default] RemoteAI = 1, } + +#[derive(Default, ProtoBuf, Validate, Clone, Debug)] +pub struct ChatFilePB { + #[pb(index = 1)] + #[validate(custom = "required_not_empty_str")] + pub file_path: String, + + #[pb(index = 2)] + #[validate(custom = "required_not_empty_str")] + pub chat_id: String, +} + +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct DownloadLLMPB { + #[pb(index = 1)] + pub progress_stream: i64, +} + +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct DownloadTaskPB { + #[pb(index = 1)] + pub task_id: String, +} +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct LocalModelStatePB { + #[pb(index = 1)] + pub model_name: String, + + #[pb(index = 2)] + pub model_size: String, + + #[pb(index = 3)] + pub need_download: bool, + + #[pb(index = 4)] + pub requirements: String, + + #[pb(index = 5)] + pub is_downloading: bool, +} + +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct LocalModelResourcePB { + #[pb(index = 1)] + pub is_ready: bool, + + #[pb(index = 2)] + pub pending_resources: Vec, + + #[pb(index = 3)] + pub is_downloading: bool, +} + +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct PendingResourcePB { + #[pb(index = 1)] + pub name: String, + + #[pb(index = 2)] + pub file_size: i64, + + #[pb(index = 3)] + pub requirements: String, +} + +#[derive(Default, ProtoBuf, Clone, Debug)] +pub struct PluginStatePB { + #[pb(index = 1)] + pub state: RunningStatePB, +} + +#[derive(Debug, Default, Clone, ProtoBuf_Enum, PartialEq, Eq, Copy)] +pub enum RunningStatePB { + #[default] + Connecting = 0, + Connected = 1, + Running = 2, + Stopped = 3, +} + +impl From for RunningStatePB { + fn from(value: RunningState) -> Self { + match value { + RunningState::Connecting => RunningStatePB::Connecting, + RunningState::Connected { .. } => RunningStatePB::Connected, + RunningState::Running { .. } => RunningStatePB::Running, + RunningState::Stopped { .. } => RunningStatePB::Stopped, + RunningState::UnexpectedStop { .. } => RunningStatePB::Stopped, + } + } +} diff --git a/frontend/rust-lib/flowy-chat/src/event_handler.rs b/frontend/rust-lib/flowy-chat/src/event_handler.rs index 924687ca5d..fead2c476b 100644 --- a/frontend/rust-lib/flowy-chat/src/event_handler.rs +++ b/frontend/rust-lib/flowy-chat/src/event_handler.rs @@ -1,14 +1,18 @@ use flowy_chat_pub::cloud::ChatMessageType; +use std::path::PathBuf; +use allo_isolate::Isolate; use std::sync::{Arc, Weak}; +use tokio::sync::oneshot; use validator::Validate; -use crate::tools::AITools; -use flowy_error::{FlowyError, FlowyResult}; -use lib_dispatch::prelude::{data_result_ok, AFPluginData, AFPluginState, DataResult}; - use crate::chat_manager::ChatManager; use crate::entities::*; +use crate::local_ai::local_llm_chat::LLMModelInfo; +use crate::tools::AITools; +use flowy_error::{FlowyError, FlowyResult}; +use lib_dispatch::prelude::{data_result_ok, AFPluginData, AFPluginState, DataResult}; +use lib_infra::isolate_stream::IsolateSink; fn upgrade_chat_manager( chat_manager: AFPluginState>, @@ -120,24 +124,44 @@ pub(crate) async fn stop_stream_handler( } #[tracing::instrument(level = "debug", skip_all, err)] -pub(crate) async fn get_local_ai_setting_handler( +pub(crate) async fn refresh_local_ai_info_handler( chat_manager: AFPluginState>, -) -> DataResult { +) -> DataResult { let chat_manager = upgrade_chat_manager(chat_manager)?; - let setting = chat_manager.get_local_ai_setting()?; - let pb = setting.into(); - data_result_ok(pb) + let (tx, rx) = oneshot::channel::>(); + tokio::spawn(async move { + let model_info = chat_manager.local_ai_controller.refresh().await; + let _ = tx.send(model_info); + }); + + let model_info = rx.await??; + data_result_ok(model_info.into()) } #[tracing::instrument(level = "debug", skip_all, err)] -pub(crate) async fn update_local_ai_setting_handler( - data: AFPluginData, +pub(crate) async fn update_local_llm_model_handler( + data: AFPluginData, chat_manager: AFPluginState>, -) -> Result<(), FlowyError> { +) -> DataResult { let data = data.into_inner(); let chat_manager = upgrade_chat_manager(chat_manager)?; - chat_manager.update_local_ai_setting(data.into())?; - Ok(()) + let state = chat_manager + .local_ai_controller + .use_local_llm(data.llm_id) + .await?; + data_result_ok(state) +} + +#[tracing::instrument(level = "debug", skip_all, err)] +pub(crate) async fn get_local_llm_state_handler( + chat_manager: AFPluginState>, +) -> DataResult { + let chat_manager = upgrade_chat_manager(chat_manager)?; + let state = chat_manager + .local_ai_controller + .get_local_llm_state() + .await?; + data_result_ok(state) } pub(crate) async fn start_complete_text_handler( @@ -157,3 +181,56 @@ pub(crate) async fn stop_complete_text_handler( tools.cancel_complete_task(&data.task_id).await; Ok(()) } + +#[tracing::instrument(level = "debug", skip_all, err)] +pub(crate) async fn chat_file_handler( + data: AFPluginData, + chat_manager: AFPluginState>, +) -> Result<(), FlowyError> { + let data = data.try_into_inner()?; + let file_path = PathBuf::from(&data.file_path); + let (tx, rx) = oneshot::channel::>(); + tokio::spawn(async move { + let chat_manager = upgrade_chat_manager(chat_manager)?; + chat_manager + .chat_with_file(&data.chat_id, file_path) + .await?; + let _ = tx.send(Ok(())); + Ok::<_, FlowyError>(()) + }); + + rx.await? +} + +#[tracing::instrument(level = "debug", skip_all, err)] +pub(crate) async fn download_llm_resource_handler( + data: AFPluginData, + chat_manager: AFPluginState>, +) -> DataResult { + let data = data.into_inner(); + let chat_manager = upgrade_chat_manager(chat_manager)?; + let text_sink = IsolateSink::new(Isolate::new(data.progress_stream)); + let task_id = chat_manager + .local_ai_controller + .start_downloading(text_sink) + .await?; + data_result_ok(DownloadTaskPB { task_id }) +} + +#[tracing::instrument(level = "debug", skip_all, err)] +pub(crate) async fn cancel_download_llm_resource_handler( + chat_manager: AFPluginState>, +) -> Result<(), FlowyError> { + let chat_manager = upgrade_chat_manager(chat_manager)?; + chat_manager.local_ai_controller.cancel_download()?; + Ok(()) +} + +#[tracing::instrument(level = "debug", skip_all, err)] +pub(crate) async fn get_plugin_state_handler( + chat_manager: AFPluginState>, +) -> DataResult { + let chat_manager = upgrade_chat_manager(chat_manager)?; + let state = chat_manager.local_ai_controller.get_plugin_state(); + data_result_ok(state) +} diff --git a/frontend/rust-lib/flowy-chat/src/event_map.rs b/frontend/rust-lib/flowy-chat/src/event_map.rs index 52aad53503..72ac807b7d 100644 --- a/frontend/rust-lib/flowy-chat/src/event_map.rs +++ b/frontend/rust-lib/flowy-chat/src/event_map.rs @@ -11,7 +11,7 @@ use crate::event_handler::*; pub fn init(chat_manager: Weak) -> AFPlugin { let user_service = Arc::downgrade(&chat_manager.upgrade().unwrap().user_service); - let cloud_service = Arc::downgrade(&chat_manager.upgrade().unwrap().chat_service); + let cloud_service = Arc::downgrade(&chat_manager.upgrade().unwrap().chat_service_wm); let ai_tools = Arc::new(AITools::new(cloud_service, user_service)); AFPlugin::new() .name("Flowy-Chat") @@ -23,13 +23,24 @@ pub fn init(chat_manager: Weak) -> AFPlugin { .event(ChatEvent::GetRelatedQuestion, get_related_question_handler) .event(ChatEvent::GetAnswerForQuestion, get_answer_handler) .event(ChatEvent::StopStream, stop_stream_handler) - .event(ChatEvent::GetLocalAISetting, get_local_ai_setting_handler) .event( - ChatEvent::UpdateLocalAISetting, - update_local_ai_setting_handler, + ChatEvent::RefreshLocalAIModelInfo, + refresh_local_ai_info_handler, ) + .event(ChatEvent::UpdateLocalLLM, update_local_llm_model_handler) + .event(ChatEvent::GetLocalLLMState, get_local_llm_state_handler) .event(ChatEvent::CompleteText, start_complete_text_handler) .event(ChatEvent::StopCompleteText, stop_complete_text_handler) + .event(ChatEvent::ChatWithFile, chat_file_handler) + .event( + ChatEvent::DownloadLLMResource, + download_llm_resource_handler, + ) + .event( + ChatEvent::CancelDownloadLLMResource, + cancel_download_llm_resource_handler, + ) + .event(ChatEvent::GetPluginState, get_plugin_state_handler) } #[derive(Clone, Copy, PartialEq, Eq, Debug, Display, Hash, ProtoBuf_Enum, Flowy_Event)] @@ -54,15 +65,30 @@ pub enum ChatEvent { #[event(input = "ChatMessageIdPB", output = "ChatMessagePB")] GetAnswerForQuestion = 5, - #[event(input = "LocalLLMSettingPB")] - UpdateLocalAISetting = 6, + #[event(input = "LLMModelPB", output = "LocalModelResourcePB")] + UpdateLocalLLM = 6, - #[event(output = "LocalLLMSettingPB")] - GetLocalAISetting = 7, + #[event(output = "LocalModelResourcePB")] + GetLocalLLMState = 7, + + #[event(output = "LLMModelInfoPB")] + RefreshLocalAIModelInfo = 8, #[event(input = "CompleteTextPB", output = "CompleteTextTaskPB")] - CompleteText = 8, + CompleteText = 9, #[event(input = "CompleteTextTaskPB")] - StopCompleteText = 9, + StopCompleteText = 10, + + #[event(input = "ChatFilePB")] + ChatWithFile = 11, + + #[event(input = "DownloadLLMPB", output = "DownloadTaskPB")] + DownloadLLMResource = 12, + + #[event()] + CancelDownloadLLMResource = 13, + + #[event(output = "PluginStatePB")] + GetPluginState = 14, } diff --git a/frontend/rust-lib/flowy-chat/src/lib.rs b/frontend/rust-lib/flowy-chat/src/lib.rs index ca5e9b1f24..2b4458d5ea 100644 --- a/frontend/rust-lib/flowy-chat/src/lib.rs +++ b/frontend/rust-lib/flowy-chat/src/lib.rs @@ -4,6 +4,7 @@ pub mod event_map; mod chat; pub mod chat_manager; pub mod entities; +mod local_ai; mod middleware; pub mod notification; mod persistence; diff --git a/frontend/rust-lib/flowy-chat/src/local_ai/llm_resource.rs b/frontend/rust-lib/flowy-chat/src/local_ai/llm_resource.rs new file mode 100644 index 0000000000..7e4d7da8a6 --- /dev/null +++ b/frontend/rust-lib/flowy-chat/src/local_ai/llm_resource.rs @@ -0,0 +1,484 @@ +use crate::chat_manager::ChatUserService; +use crate::entities::{LocalModelResourcePB, PendingResourcePB}; +use crate::local_ai::local_llm_chat::{LLMModelInfo, LLMSetting}; +use crate::local_ai::model_request::download_model; + +use appflowy_local_ai::chat_plugin::AIPluginConfig; +use flowy_chat_pub::cloud::{LLMModel, LocalAIConfig, ModelInfo}; +use flowy_error::{FlowyError, FlowyResult}; +use futures::Sink; +use futures_util::SinkExt; +use lib_infra::async_trait::async_trait; +use parking_lot::RwLock; + +use appflowy_local_ai::plugin_request::download_plugin; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::fs::{self}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, instrument, trace}; +use zip_extensions::zip_extract; + +#[async_trait] +pub trait LLMResourceService: Send + Sync + 'static { + async fn get_local_ai_config(&self) -> Result; + fn store(&self, setting: LLMSetting) -> Result<(), anyhow::Error>; + fn retrieve(&self) -> Option; +} + +const PLUGIN_DIR: &str = "plugin"; +const LLM_MODEL_DIR: &str = "models"; +const DOWNLOAD_FINISH: &str = "finish"; + +pub enum PendingResource { + PluginRes, + ModelInfoRes(Vec), +} +#[derive(Clone)] +pub struct DownloadTask { + cancel_token: CancellationToken, + tx: tokio::sync::broadcast::Sender, +} +impl DownloadTask { + pub fn new() -> Self { + let (tx, _) = tokio::sync::broadcast::channel(5); + let cancel_token = CancellationToken::new(); + Self { cancel_token, tx } + } + + pub fn cancel(&self) { + self.cancel_token.cancel(); + } +} + +pub struct LLMResourceController { + user_service: Arc, + resource_service: Arc, + llm_setting: RwLock>, + // The ai_config will be set when user try to get latest local ai config from server + ai_config: RwLock>, + download_task: Arc>>, + resource_notify: tokio::sync::mpsc::Sender<()>, +} + +impl LLMResourceController { + pub fn new( + user_service: Arc, + resource_service: impl LLMResourceService, + resource_notify: tokio::sync::mpsc::Sender<()>, + ) -> Self { + let llm_setting = RwLock::new(resource_service.retrieve()); + Self { + user_service, + resource_service: Arc::new(resource_service), + llm_setting, + ai_config: Default::default(), + download_task: Default::default(), + resource_notify, + } + } + + /// Returns true when all resources are downloaded and ready to use. + pub fn is_resource_ready(&self) -> bool { + match self.calculate_pending_resources() { + Ok(res) => res.is_empty(), + Err(_) => false, + } + } + + /// Retrieves model information and updates the current model settings. + #[instrument(level = "debug", skip_all, err)] + pub async fn refresh_llm_resource(&self) -> FlowyResult { + let ai_config = self.fetch_ai_config().await?; + if ai_config.models.is_empty() { + return Err(FlowyError::local_ai().with_context("No model found")); + } + + *self.ai_config.write() = Some(ai_config.clone()); + let selected_model = self.select_model(&ai_config)?; + + let llm_setting = LLMSetting { + plugin: ai_config.plugin.clone(), + llm_model: selected_model.clone(), + }; + self.llm_setting.write().replace(llm_setting.clone()); + self.resource_service.store(llm_setting)?; + + Ok(LLMModelInfo { + selected_model, + models: ai_config.models, + }) + } + + #[instrument(level = "info", skip_all, err)] + pub fn use_local_llm(&self, llm_id: i64) -> FlowyResult { + let (package, llm_config) = self + .ai_config + .read() + .as_ref() + .and_then(|config| { + config + .models + .iter() + .find(|model| model.llm_id == llm_id) + .cloned() + .map(|model| (config.plugin.clone(), model)) + }) + .ok_or_else(|| FlowyError::local_ai().with_context("No local ai config found"))?; + + let llm_setting = LLMSetting { + plugin: package, + llm_model: llm_config.clone(), + }; + + trace!("[LLM Resource] Selected AI setting: {:?}", llm_setting); + *self.llm_setting.write() = Some(llm_setting.clone()); + self.resource_service.store(llm_setting)?; + self.get_local_llm_state() + } + + pub fn get_local_llm_state(&self) -> FlowyResult { + let state = self + .check_resource() + .ok_or_else(|| FlowyError::local_ai().with_context("No local ai config found"))?; + Ok(state) + } + + #[instrument(level = "debug", skip_all)] + fn check_resource(&self) -> Option { + trace!("[LLM Resource] Checking local ai resources"); + + let pending_resources = self.calculate_pending_resources().ok()?; + let is_ready = pending_resources.is_empty(); + let is_downloading = self.download_task.read().is_some(); + let pending_resources: Vec<_> = pending_resources + .into_iter() + .flat_map(|res| match res { + PendingResource::PluginRes => vec![PendingResourcePB { + name: "AppFlowy Plugin".to_string(), + file_size: 0, + requirements: "".to_string(), + }], + PendingResource::ModelInfoRes(model_infos) => model_infos + .into_iter() + .map(|model_info| PendingResourcePB { + name: model_info.name, + file_size: model_info.file_size, + requirements: model_info.requirements, + }) + .collect::>(), + }) + .collect(); + + let resource = LocalModelResourcePB { + is_ready, + pending_resources, + is_downloading, + }; + + debug!("[LLM Resource] Local AI resources state: {:?}", resource); + Some(resource) + } + + /// Returns true when all resources are downloaded and ready to use. + pub fn calculate_pending_resources(&self) -> FlowyResult> { + match self.llm_setting.read().as_ref() { + None => Err(FlowyError::local_ai().with_context("Can't find any llm config")), + Some(llm_setting) => { + let mut resources = vec![]; + let plugin_path = self.plugin_path(&llm_setting.plugin.etag)?; + + if !plugin_path.exists() { + trace!("[LLM Resource] Plugin file not found: {:?}", plugin_path); + resources.push(PendingResource::PluginRes); + } + + let chat_model = self.model_path(&llm_setting.llm_model.chat_model.file_name)?; + if !chat_model.exists() { + resources.push(PendingResource::ModelInfoRes(vec![llm_setting + .llm_model + .chat_model + .clone()])); + } + + let embedding_model = self.model_path(&llm_setting.llm_model.embedding_model.file_name)?; + if !embedding_model.exists() { + resources.push(PendingResource::ModelInfoRes(vec![llm_setting + .llm_model + .embedding_model + .clone()])); + } + + Ok(resources) + }, + } + } + + #[instrument(level = "info", skip_all, err)] + pub async fn start_downloading(&self, mut progress_sink: T) -> FlowyResult + where + T: Sink + Unpin + Sync + Send + 'static, + { + let task_id = uuid::Uuid::new_v4().to_string(); + let weak_download_task = Arc::downgrade(&self.download_task); + let resource_notify = self.resource_notify.clone(); + // notify download progress to client. + let progress_notify = |mut rx: tokio::sync::broadcast::Receiver| { + tokio::spawn(async move { + while let Ok(value) = rx.recv().await { + let is_finish = value == DOWNLOAD_FINISH; + if let Err(err) = progress_sink.send(value).await { + error!("Failed to send progress: {:?}", err); + break; + } + + if is_finish { + info!("notify download finish, need to reload resources"); + let _ = resource_notify.send(()).await; + if let Some(download_task) = weak_download_task.upgrade() { + if let Some(task) = download_task.write().take() { + task.cancel(); + } + } + break; + } + } + }); + }; + + // return immediately if download task already exists + if let Some(download_task) = self.download_task.read().as_ref() { + trace!( + "Download task already exists, return the task id: {}", + task_id + ); + progress_notify(download_task.tx.subscribe()); + return Ok(task_id); + } + + // If download task is not exists, create a new download task. + info!("[LLM Resource] Start new download task"); + let llm_setting = self + .llm_setting + .read() + .clone() + .ok_or_else(|| FlowyError::local_ai().with_context("No local ai config found"))?; + + let download_task = DownloadTask::new(); + *self.download_task.write() = Some(download_task.clone()); + progress_notify(download_task.tx.subscribe()); + + let plugin_dir = self.user_plugin_folder()?; + if !plugin_dir.exists() { + fs::create_dir_all(&plugin_dir).await.map_err(|err| { + FlowyError::local_ai().with_context(format!("Failed to create plugin dir: {:?}", err)) + })?; + } + + let model_dir = self.user_model_folder()?; + if !model_dir.exists() { + fs::create_dir_all(&model_dir).await.map_err(|err| { + FlowyError::local_ai().with_context(format!("Failed to create model dir: {:?}", err)) + })?; + } + + tokio::spawn(async move { + let plugin_file_etag_dir = plugin_dir.join(&llm_setting.plugin.etag); + // We use the ETag as the identifier for the plugin file. If a file with the given ETag + // already exists, skip downloading it. + if !plugin_file_etag_dir.exists() { + let plugin_progress_tx = download_task.tx.clone(); + info!( + "[LLM Resource] Downloading plugin: {:?}", + llm_setting.plugin.etag + ); + let file_name = format!("{}.zip", llm_setting.plugin.etag); + let zip_plugin_file = download_plugin( + &llm_setting.plugin.url, + &plugin_dir, + &file_name, + Some(download_task.cancel_token.clone()), + Some(Arc::new(move |downloaded, total_size| { + let progress = (downloaded as f64 / total_size as f64).clamp(0.0, 1.0); + let _ = plugin_progress_tx.send(format!("plugin:progress:{}", progress)); + })), + ) + .await?; + + // unzip file + info!( + "[LLM Resource] unzip {:?} to {:?}", + zip_plugin_file, plugin_file_etag_dir + ); + zip_extract(&zip_plugin_file, &plugin_file_etag_dir)?; + + // delete zip file + info!("[LLM Resource] Delete zip file: {:?}", file_name); + if let Err(err) = fs::remove_file(&zip_plugin_file).await { + error!("Failed to delete zip file: {:?}", err); + } + } + + // After download the plugin, start downloading models + let chat_model_file = ( + model_dir.join(&llm_setting.llm_model.chat_model.file_name), + llm_setting.llm_model.chat_model.file_name, + llm_setting.llm_model.chat_model.name, + llm_setting.llm_model.chat_model.download_url, + ); + let embedding_model_file = ( + model_dir.join(&llm_setting.llm_model.embedding_model.file_name), + llm_setting.llm_model.embedding_model.file_name, + llm_setting.llm_model.embedding_model.name, + llm_setting.llm_model.embedding_model.download_url, + ); + for (file_path, file_name, model_name, url) in [chat_model_file, embedding_model_file] { + if file_path.exists() { + continue; + } + + info!("[LLM Resource] Downloading model: {:?}", file_name); + let plugin_progress_tx = download_task.tx.clone(); + let cloned_model_name = model_name.clone(); + let progress = Arc::new(move |downloaded, total_size| { + let progress = (downloaded as f64 / total_size as f64).clamp(0.0, 1.0); + let _ = plugin_progress_tx.send(format!("{}:progress:{}", cloned_model_name, progress)); + }); + match download_model( + &url, + &model_dir, + &file_name, + Some(progress), + Some(download_task.cancel_token.clone()), + ) + .await + { + Ok(_) => info!("[LLM Resource] Downloaded model: {:?}", file_name), + Err(err) => { + error!( + "[LLM Resource] Failed to download model for given url: {:?}, error: {:?}", + url, err + ); + download_task + .tx + .send(format!("error:failed to download {}", model_name))?; + continue; + }, + } + } + info!("[LLM Resource] All resources downloaded"); + download_task.tx.send(DOWNLOAD_FINISH.to_string())?; + Ok::<_, anyhow::Error>(()) + }); + + Ok(task_id) + } + + pub fn cancel_download(&self) -> FlowyResult<()> { + if let Some(cancel_token) = self.download_task.write().take() { + info!("[LLM Resource] Cancel download"); + cancel_token.cancel(); + } + + Ok(()) + } + + #[instrument(level = "debug", skip_all, err)] + pub fn get_ai_plugin_config(&self) -> FlowyResult { + if !self.is_resource_ready() { + return Err(FlowyError::local_ai().with_context("Local AI resources are not ready")); + } + + let llm_setting = self + .llm_setting + .read() + .as_ref() + .cloned() + .ok_or_else(|| FlowyError::local_ai().with_context("No local llm setting found"))?; + + let model_dir = self.user_model_folder()?; + let resource_dir = self.resource_dir()?; + + let bin_path = self + .plugin_path(&llm_setting.plugin.etag)? + .join(llm_setting.plugin.name); + let chat_model_path = model_dir.join(&llm_setting.llm_model.chat_model.file_name); + let embedding_model_path = model_dir.join(&llm_setting.llm_model.embedding_model.file_name); + let mut config = AIPluginConfig::new(bin_path, chat_model_path)?; + + // + let persist_directory = resource_dir.join("rag"); + if !persist_directory.exists() { + std::fs::create_dir_all(&persist_directory)?; + } + + // Enable RAG when the embedding model path is set + config.set_rag_enabled(&embedding_model_path, &persist_directory)?; + + if cfg!(debug_assertions) { + config = config.with_verbose(true); + } + Ok(config) + } + + /// Fetches the local AI configuration from the resource service. + async fn fetch_ai_config(&self) -> FlowyResult { + self + .resource_service + .get_local_ai_config() + .await + .map_err(|err| { + error!("[LLM Resource] Failed to fetch local ai config: {:?}", err); + FlowyError::local_ai() + .with_context("Can't retrieve model info. Please try again later".to_string()) + }) + } + + /// Selects the appropriate model based on the current settings or defaults to the first model. + fn select_model(&self, ai_config: &LocalAIConfig) -> FlowyResult { + let selected_model = match self.llm_setting.read().as_ref() { + None => ai_config.models[0].clone(), + Some(llm_setting) => { + match ai_config + .models + .iter() + .find(|model| model.llm_id == llm_setting.llm_model.llm_id) + { + None => ai_config.models[0].clone(), + Some(llm_model) => { + if llm_model != &llm_setting.llm_model { + info!( + "[LLM Resource] existing model is different from remote, replace with remote model" + ); + } + llm_model.clone() + }, + } + }, + }; + Ok(selected_model) + } + + fn user_plugin_folder(&self) -> FlowyResult { + self.resource_dir().map(|dir| dir.join(PLUGIN_DIR)) + } + + fn user_model_folder(&self) -> FlowyResult { + self.resource_dir().map(|dir| dir.join(LLM_MODEL_DIR)) + } + + fn plugin_path(&self, etag: &str) -> FlowyResult { + self.user_plugin_folder().map(|dir| dir.join(etag)) + } + + fn model_path(&self, model_file_name: &str) -> FlowyResult { + self + .user_model_folder() + .map(|dir| dir.join(model_file_name)) + } + + fn resource_dir(&self) -> FlowyResult { + let user_data_dir = self.user_service.user_data_dir()?; + Ok(user_data_dir.join("llm")) + } +} diff --git a/frontend/rust-lib/flowy-chat/src/local_ai/local_llm_chat.rs b/frontend/rust-lib/flowy-chat/src/local_ai/local_llm_chat.rs new file mode 100644 index 0000000000..0b0aa68d52 --- /dev/null +++ b/frontend/rust-lib/flowy-chat/src/local_ai/local_llm_chat.rs @@ -0,0 +1,254 @@ +use crate::chat_manager::ChatUserService; +use crate::entities::{ + ChatStatePB, LocalModelResourcePB, ModelTypePB, PluginStatePB, RunningStatePB, +}; +use crate::local_ai::llm_resource::{LLMResourceController, LLMResourceService}; +use crate::notification::{send_notification, ChatNotification}; +use anyhow::Error; +use appflowy_local_ai::chat_plugin::{AIPluginConfig, LocalChatLLMChat}; +use appflowy_plugin::manager::PluginManager; +use appflowy_plugin::util::is_apple_silicon; +use flowy_chat_pub::cloud::{AppFlowyAIPlugin, ChatCloudService, LLMModel, LocalAIConfig}; +use flowy_error::FlowyResult; +use flowy_sqlite::kv::KVStorePreferences; +use futures::Sink; +use lib_infra::async_trait::async_trait; + +use serde::{Deserialize, Serialize}; +use std::ops::Deref; + +use parking_lot::Mutex; +use std::sync::Arc; +use tokio_stream::StreamExt; +use tracing::{debug, error, info, trace}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LLMSetting { + pub plugin: AppFlowyAIPlugin, + pub llm_model: LLMModel, +} + +pub struct LLMModelInfo { + pub selected_model: LLMModel, + pub models: Vec, +} + +const LOCAL_AI_SETTING_KEY: &str = "local_ai_setting"; +pub struct LocalAIController { + llm_chat: Arc, + llm_res: Arc, + current_chat_id: Mutex>, +} + +impl Deref for LocalAIController { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.llm_chat + } +} + +impl LocalAIController { + pub fn new( + plugin_manager: Arc, + store_preferences: Arc, + user_service: Arc, + cloud_service: Arc, + ) -> Self { + let llm_chat = Arc::new(LocalChatLLMChat::new(plugin_manager)); + let mut rx = llm_chat.subscribe_running_state(); + tokio::spawn(async move { + while let Some(state) = rx.next().await { + let new_state = RunningStatePB::from(state); + info!("[AI Plugin] state: {:?}", new_state); + send_notification( + "appflowy_chat_plugin", + ChatNotification::UpdateChatPluginState, + ) + .payload(PluginStatePB { state: new_state }) + .send(); + } + }); + + let res_impl = LLMResourceServiceImpl { + user_service: user_service.clone(), + cloud_service, + store_preferences, + }; + + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let llm_res = Arc::new(LLMResourceController::new(user_service, res_impl, tx)); + + let cloned_llm_chat = llm_chat.clone(); + let cloned_llm_res = llm_res.clone(); + tokio::spawn(async move { + while rx.recv().await.is_some() { + if let Ok(chat_config) = cloned_llm_res.get_ai_plugin_config() { + initialize_chat_plugin(&cloned_llm_chat, chat_config).unwrap(); + } + } + }); + + Self { + llm_chat, + llm_res, + current_chat_id: Default::default(), + } + } + pub async fn refresh(&self) -> FlowyResult { + self.llm_res.refresh_llm_resource().await + } + + pub fn initialize(&self) -> FlowyResult<()> { + let chat_config = self.llm_res.get_ai_plugin_config()?; + let llm_chat = self.llm_chat.clone(); + initialize_chat_plugin(&llm_chat, chat_config)?; + Ok(()) + } + + /// Returns true if the local AI is enabled and ready to use. + pub fn is_ready(&self) -> bool { + self.llm_res.is_resource_ready() + } + + pub fn open_chat(&self, chat_id: &str) { + if !self.is_ready() { + return; + } + + // Only keep one chat open at a time. Since loading multiple models at the same time will cause + // memory issues. + if let Some(current_chat_id) = self.current_chat_id.lock().as_ref() { + debug!("[AI Plugin] close previous chat: {}", current_chat_id); + self.close_chat(current_chat_id); + } + + *self.current_chat_id.lock() = Some(chat_id.to_string()); + let chat_id = chat_id.to_string(); + let weak_ctrl = Arc::downgrade(&self.llm_chat); + tokio::spawn(async move { + if let Some(ctrl) = weak_ctrl.upgrade() { + if let Err(err) = ctrl.create_chat(&chat_id).await { + error!("[AI Plugin] failed to open chat: {:?}", err); + } + } + }); + } + + pub fn close_chat(&self, chat_id: &str) { + let weak_ctrl = Arc::downgrade(&self.llm_chat); + let chat_id = chat_id.to_string(); + tokio::spawn(async move { + if let Some(ctrl) = weak_ctrl.upgrade() { + if let Err(err) = ctrl.close_chat(&chat_id).await { + error!("[AI Plugin] failed to close chat: {:?}", err); + } + } + }); + } + + pub async fn use_local_llm(&self, llm_id: i64) -> FlowyResult { + let llm_chat = self.llm_chat.clone(); + match llm_chat.destroy_chat_plugin().await { + Ok(_) => info!("[AI Plugin] destroy plugin successfully"), + Err(err) => error!("[AI Plugin] failed to destroy plugin: {:?}", err), + } + let state = self.llm_res.use_local_llm(llm_id)?; + // Re-initialize the plugin if the setting is updated and ready to use + if self.llm_res.is_resource_ready() { + self.initialize()?; + } + Ok(state) + } + + pub async fn get_local_llm_state(&self) -> FlowyResult { + self.llm_res.get_local_llm_state() + } + + pub async fn start_downloading(&self, progress_sink: T) -> FlowyResult + where + T: Sink + Unpin + Sync + Send + 'static, + { + let task_id = self.llm_res.start_downloading(progress_sink).await?; + Ok(task_id) + } + + pub fn cancel_download(&self) -> FlowyResult<()> { + self.llm_res.cancel_download()?; + Ok(()) + } + + pub fn get_plugin_state(&self) -> PluginStatePB { + let state = self.llm_chat.get_plugin_running_state(); + PluginStatePB { + state: RunningStatePB::from(state), + } + } +} + +fn initialize_chat_plugin( + llm_chat: &Arc, + mut chat_config: AIPluginConfig, +) -> FlowyResult<()> { + let llm_chat = llm_chat.clone(); + tokio::spawn(async move { + trace!("[AI Plugin] config: {:?}", chat_config); + if is_apple_silicon().await.unwrap_or(false) { + chat_config = chat_config.with_device("gpu"); + } + match llm_chat.init_chat_plugin(chat_config).await { + Ok(_) => { + send_notification( + "appflowy_chat_plugin", + ChatNotification::UpdateChatPluginState, + ) + .payload(ChatStatePB { + model_type: ModelTypePB::LocalAI, + available: true, + }); + }, + Err(err) => { + send_notification( + "appflowy_chat_plugin", + ChatNotification::UpdateChatPluginState, + ) + .payload(ChatStatePB { + model_type: ModelTypePB::LocalAI, + available: false, + }); + error!("[AI Plugin] failed to setup plugin: {:?}", err); + }, + } + }); + Ok(()) +} + +pub struct LLMResourceServiceImpl { + user_service: Arc, + cloud_service: Arc, + store_preferences: Arc, +} +#[async_trait] +impl LLMResourceService for LLMResourceServiceImpl { + async fn get_local_ai_config(&self) -> Result { + let workspace_id = self.user_service.workspace_id()?; + let config = self + .cloud_service + .get_local_ai_config(&workspace_id) + .await?; + Ok(config) + } + + fn store(&self, setting: LLMSetting) -> Result<(), Error> { + self + .store_preferences + .set_object(LOCAL_AI_SETTING_KEY, setting)?; + Ok(()) + } + + fn retrieve(&self) -> Option { + self + .store_preferences + .get_object::(LOCAL_AI_SETTING_KEY) + } +} diff --git a/frontend/rust-lib/flowy-chat/src/local_ai/mod.rs b/frontend/rust-lib/flowy-chat/src/local_ai/mod.rs new file mode 100644 index 0000000000..cb94a2dc3e --- /dev/null +++ b/frontend/rust-lib/flowy-chat/src/local_ai/mod.rs @@ -0,0 +1,3 @@ +pub mod llm_resource; +pub mod local_llm_chat; +mod model_request; diff --git a/frontend/rust-lib/flowy-chat/src/local_ai/model_request.rs b/frontend/rust-lib/flowy-chat/src/local_ai/model_request.rs new file mode 100644 index 0000000000..1692b6ce57 --- /dev/null +++ b/frontend/rust-lib/flowy-chat/src/local_ai/model_request.rs @@ -0,0 +1,149 @@ +use anyhow::{anyhow, Result}; +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use reqwest::{Client, Response, StatusCode}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::fs::{self, File}; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +use tokio_util::sync::CancellationToken; +use tracing::{instrument, trace}; + +type ProgressCallback = Arc; + +#[instrument(level = "trace", skip_all, err)] +pub async fn download_model( + url: &str, + model_path: &Path, + model_filename: &str, + progress_callback: Option, + cancel_token: Option, +) -> Result { + let client = Client::new(); + let mut response = make_request(&client, url, None).await?; + let total_size_in_bytes = response.content_length().unwrap_or(0); + let partial_path = model_path.join(format!("{}.part", model_filename)); + let download_path = model_path.join(model_filename); + let mut part_file = File::create(&partial_path).await?; + let mut downloaded: u64 = 0; + + while let Some(chunk) = response.chunk().await? { + if let Some(cancel_token) = &cancel_token { + if cancel_token.is_cancelled() { + trace!("Download canceled by client"); + fs::remove_file(&partial_path).await?; + return Err(anyhow!("Download canceled")); + } + } + + part_file.write_all(&chunk).await?; + downloaded += chunk.len() as u64; + + if let Some(progress_callback) = &progress_callback { + progress_callback(downloaded, total_size_in_bytes); + } + } + + // Verify file integrity + let header_sha256 = response + .headers() + .get("SHA256") + .and_then(|value| value.to_str().ok()) + .and_then(|value| STANDARD.decode(value).ok()); + + part_file.seek(tokio::io::SeekFrom::Start(0)).await?; + let mut hasher = Sha256::new(); + let block_size = 2_usize.pow(20); // 1 MB + let mut buffer = vec![0; block_size]; + while let Ok(bytes_read) = part_file.read(&mut buffer).await { + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + } + let calculated_sha256 = hasher.finalize(); + if let Some(header_sha256) = header_sha256 { + if calculated_sha256.as_slice() != header_sha256.as_slice() { + trace!( + "Header Sha256: {:?}, calculated Sha256:{:?}", + header_sha256, + calculated_sha256 + ); + + fs::remove_file(&partial_path).await?; + return Err(anyhow!( + "Sha256 mismatch: expected {:?}, got {:?}", + header_sha256, + calculated_sha256 + )); + } + } + + fs::rename(&partial_path, &download_path).await?; + Ok(download_path) +} + +async fn make_request( + client: &Client, + url: &str, + offset: Option, +) -> Result { + let mut request = client.get(url); + if let Some(offset) = offset { + println!( + "\nDownload interrupted, resuming from byte position {}", + offset + ); + request = request.header("Range", format!("bytes={}-", offset)); + } + let response = request.send().await?; + if !(response.status().is_success() || response.status() == StatusCode::PARTIAL_CONTENT) { + return Err(anyhow!(response.text().await?)); + } + Ok(response) +} + +#[cfg(test)] +mod test { + use super::*; + use std::env::temp_dir; + #[tokio::test] + async fn retrieve_gpt4all_model_test() { + for url in [ + "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf", + "https://huggingface.co/second-state/All-MiniLM-L6-v2-Embedding-GGUF/resolve/main/all-MiniLM-L6-v2-Q3_K_L.gguf?download=true", + // "https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.Q4_K_M.gguf?download=true", + ] { + let temp_dir = temp_dir().join("download_llm"); + if !temp_dir.exists() { + fs::create_dir(&temp_dir).await.unwrap(); + } + let file_name = "llm_model.gguf"; + let cancel_token = CancellationToken::new(); + let token = cancel_token.clone(); + tokio::spawn(async move { + tokio::time::sleep(tokio::time::Duration::from_secs(30)).await; + token.cancel(); + }); + + let download_file = download_model( + url, + &temp_dir, + file_name, + Some(Arc::new(|a, b| { + println!("{}/{}", a, b); + })), + Some(cancel_token), + ).await.unwrap(); + + let file_path = temp_dir.join(file_name); + assert_eq!(download_file, file_path); + + println!("File path: {:?}", file_path); + assert!(file_path.exists()); + std::fs::remove_file(file_path).unwrap(); + } + } +} diff --git a/frontend/rust-lib/flowy-chat/src/middleware/chat_service_mw.rs b/frontend/rust-lib/flowy-chat/src/middleware/chat_service_mw.rs index e3b42a8904..1ca4160ea8 100644 --- a/frontend/rust-lib/flowy-chat/src/middleware/chat_service_mw.rs +++ b/frontend/rust-lib/flowy-chat/src/middleware/chat_service_mw.rs @@ -1,97 +1,41 @@ use crate::chat_manager::ChatUserService; use crate::entities::{ChatStatePB, ModelTypePB}; +use crate::local_ai::local_llm_chat::LocalAIController; use crate::notification::{send_notification, ChatNotification}; use crate::persistence::select_single_message; -use appflowy_local_ai::llm_chat::{LocalChatLLMChat, LocalLLMSetting}; use appflowy_plugin::error::PluginError; -use appflowy_plugin::util::is_apple_silicon; + use flowy_chat_pub::cloud::{ - ChatCloudService, ChatMessage, ChatMessageType, CompletionType, MessageCursor, + ChatCloudService, ChatMessage, ChatMessageType, CompletionType, LocalAIConfig, MessageCursor, RepeatedChatMessage, RepeatedRelatedQuestion, StreamAnswer, StreamComplete, }; use flowy_error::{FlowyError, FlowyResult}; use futures::{stream, StreamExt, TryStreamExt}; use lib_infra::async_trait::async_trait; use lib_infra::future::FutureResult; -use parking_lot::RwLock; -use std::sync::Arc; -use tracing::{error, info, trace}; -pub struct ChatService { +use std::path::PathBuf; +use std::sync::Arc; + +pub struct ChatServiceMiddleware { pub cloud_service: Arc, user_service: Arc, - local_llm_chat: Arc, - local_llm_setting: Arc>, + local_llm_controller: Arc, } -impl ChatService { +impl ChatServiceMiddleware { pub fn new( user_service: Arc, cloud_service: Arc, - local_llm_ctrl: Arc, - local_llm_setting: LocalLLMSetting, + local_llm_controller: Arc, ) -> Self { - if local_llm_setting.enabled { - setup_local_chat(&local_llm_setting, local_llm_ctrl.clone()); - } - - let mut rx = local_llm_ctrl.subscribe_running_state(); - tokio::spawn(async move { - while let Ok(state) = rx.recv().await { - info!("[Chat Plugin] state: {:?}", state); - } - }); - Self { user_service, cloud_service, - local_llm_chat: local_llm_ctrl, - local_llm_setting: Arc::new(RwLock::new(local_llm_setting)), + local_llm_controller, } } - pub fn notify_open_chat(&self, chat_id: &str) { - if self.local_llm_setting.read().enabled { - trace!("[Chat Plugin] notify open chat: {}", chat_id); - let chat_id = chat_id.to_string(); - let weak_ctrl = Arc::downgrade(&self.local_llm_chat); - tokio::spawn(async move { - if let Some(ctrl) = weak_ctrl.upgrade() { - if let Err(err) = ctrl.create_chat(&chat_id).await { - error!("[Chat Plugin] failed to open chat: {:?}", err); - } - } - }); - } - } - - pub fn notify_close_chat(&self, chat_id: &str) { - if self.local_llm_setting.read().enabled { - trace!("[Chat Plugin] notify close chat: {}", chat_id); - let weak_ctrl = Arc::downgrade(&self.local_llm_chat); - let chat_id = chat_id.to_string(); - tokio::spawn(async move { - if let Some(ctrl) = weak_ctrl.upgrade() { - if let Err(err) = ctrl.close_chat(&chat_id).await { - error!("[Chat Plugin] failed to close chat: {:?}", err); - } - } - }); - } - } - - pub fn get_local_ai_setting(&self) -> LocalLLMSetting { - self.local_llm_setting.read().clone() - } - - pub fn update_local_ai_setting(&self, setting: LocalLLMSetting) -> FlowyResult<()> { - setting.validate()?; - - setup_local_chat(&setting, self.local_llm_chat.clone()); - *self.local_llm_setting.write() = setting; - Ok(()) - } - fn get_message_content(&self, message_id: i64) -> FlowyResult { let uid = self.user_service.user_id()?; let conn = self.user_service.sqlite_connection(uid)?; @@ -109,18 +53,20 @@ impl ChatService { err, PluginError::PluginNotConnected | PluginError::PeerDisconnect ) { - send_notification("appflowy_chat_plugin", ChatNotification::ChatStateUpdated).payload( - ChatStatePB { - model_type: ModelTypePB::LocalAI, - available: false, - }, - ); + send_notification( + "appflowy_chat_plugin", + ChatNotification::UpdateChatPluginState, + ) + .payload(ChatStatePB { + model_type: ModelTypePB::LocalAI, + available: false, + }); } } } #[async_trait] -impl ChatCloudService for ChatService { +impl ChatCloudService for ChatServiceMiddleware { fn create_chat( &self, uid: &i64, @@ -160,9 +106,13 @@ impl ChatCloudService for ChatService { chat_id: &str, message_id: i64, ) -> Result { - if self.local_llm_setting.read().enabled { + if self.local_llm_controller.is_ready() { let content = self.get_message_content(message_id)?; - match self.local_llm_chat.stream_question(chat_id, &content).await { + match self + .local_llm_controller + .stream_question(chat_id, &content) + .await + { Ok(stream) => Ok( stream .map_err(|err| FlowyError::local_ai().with_context(err)) @@ -187,9 +137,13 @@ impl ChatCloudService for ChatService { chat_id: &str, question_message_id: i64, ) -> Result { - if self.local_llm_setting.read().enabled { + if self.local_llm_controller.is_ready() { let content = self.get_message_content(question_message_id)?; - match self.local_llm_chat.ask_question(chat_id, &content).await { + match self + .local_llm_controller + .ask_question(chat_id, &content) + .await + { Ok(answer) => { let message = self .cloud_service @@ -228,7 +182,7 @@ impl ChatCloudService for ChatService { chat_id: &str, message_id: i64, ) -> FutureResult { - if self.local_llm_setting.read().enabled { + if self.local_llm_controller.is_ready() { FutureResult::new(async move { Ok(RepeatedRelatedQuestion { message_id, @@ -248,8 +202,10 @@ impl ChatCloudService for ChatService { text: &str, complete_type: CompletionType, ) -> Result { - if self.local_llm_setting.read().enabled { - todo!() + if self.local_llm_controller.is_ready() { + return Err( + FlowyError::not_support().with_context("completion with local ai is not supported yet"), + ); } else { self .cloud_service @@ -257,48 +213,29 @@ impl ChatCloudService for ChatService { .await } } -} -fn setup_local_chat(local_llm_setting: &LocalLLMSetting, llm_chat_ctrl: Arc) { - if local_llm_setting.enabled { - if let Ok(mut config) = local_llm_setting.chat_config() { - tokio::spawn(async move { - trace!("[Chat Plugin] setup local chat: {:?}", config); - if is_apple_silicon().await.unwrap_or(false) { - config = config.with_device("gpu"); - } - - if cfg!(debug_assertions) { - config = config.with_verbose(true); - } - - match llm_chat_ctrl.init_chat_plugin(config).await { - Ok(_) => { - send_notification("appflowy_chat_plugin", ChatNotification::ChatStateUpdated).payload( - ChatStatePB { - model_type: ModelTypePB::LocalAI, - available: true, - }, - ); - }, - Err(err) => { - send_notification("appflowy_chat_plugin", ChatNotification::ChatStateUpdated).payload( - ChatStatePB { - model_type: ModelTypePB::LocalAI, - available: false, - }, - ); - error!("[Chat Plugin] failed to setup plugin: {:?}", err); - }, - } - }); + async fn index_file( + &self, + workspace_id: &str, + file_path: PathBuf, + chat_id: &str, + ) -> Result<(), FlowyError> { + if self.local_llm_controller.is_ready() { + self + .local_llm_controller + .index_file(chat_id, file_path) + .await + .map_err(|err| FlowyError::local_ai().with_context(err))?; + Ok(()) + } else { + self + .cloud_service + .index_file(workspace_id, file_path, chat_id) + .await } - } else { - tokio::spawn(async move { - match llm_chat_ctrl.destroy_chat_plugin().await { - Ok(_) => info!("[Chat Plugin] destroy plugin successfully"), - Err(err) => error!("[Chat Plugin] failed to destroy plugin: {:?}", err), - } - }); + } + + async fn get_local_ai_config(&self, workspace_id: &str) -> Result { + self.cloud_service.get_local_ai_config(workspace_id).await } } diff --git a/frontend/rust-lib/flowy-chat/src/notification.rs b/frontend/rust-lib/flowy-chat/src/notification.rs index 10101027ae..98c20a47c3 100644 --- a/frontend/rust-lib/flowy-chat/src/notification.rs +++ b/frontend/rust-lib/flowy-chat/src/notification.rs @@ -12,7 +12,8 @@ pub enum ChatNotification { DidReceiveChatMessage = 3, StreamChatMessageError = 4, FinishStreaming = 5, - ChatStateUpdated = 6, + UpdateChatPluginState = 6, + LocalAIResourceNeeded = 7, } impl std::convert::From for i32 { @@ -28,7 +29,8 @@ impl std::convert::From for ChatNotification { 3 => ChatNotification::DidReceiveChatMessage, 4 => ChatNotification::StreamChatMessageError, 5 => ChatNotification::FinishStreaming, - 6 => ChatNotification::ChatStateUpdated, + 6 => ChatNotification::UpdateChatPluginState, + 7 => ChatNotification::LocalAIResourceNeeded, _ => ChatNotification::Unknown, } } diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs index 32a31fb274..be503e4afd 100644 --- a/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs +++ b/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs @@ -4,6 +4,7 @@ use flowy_error::FlowyError; use flowy_sqlite::kv::KVStorePreferences; use flowy_sqlite::DBConnection; use flowy_user::services::authenticate_user::AuthenticateUser; +use std::path::PathBuf; use std::sync::{Arc, Weak}; pub struct ChatDepsResolver; @@ -50,4 +51,8 @@ impl ChatUserService for ChatUserServiceImpl { fn sqlite_connection(&self, uid: i64) -> Result { self.upgrade_user()?.get_sqlite_connection(uid) } + + fn user_data_dir(&self) -> Result { + self.upgrade_user()?.get_user_data_dir() + } } diff --git a/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs b/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs index 80fd3c2523..a197bc1708 100644 --- a/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs +++ b/frontend/rust-lib/flowy-core/src/integrate/trait_impls.rs @@ -1,5 +1,6 @@ use client_api::entity::search_dto::SearchDocumentResponseItem; use flowy_search_pub::cloud::SearchCloudService; +use std::path::PathBuf; use std::sync::Arc; use anyhow::Error; @@ -18,7 +19,8 @@ use collab_integrate::collab_builder::{ CollabCloudPluginProvider, CollabPluginProviderContext, CollabPluginProviderType, }; use flowy_chat_pub::cloud::{ - ChatCloudService, ChatMessage, MessageCursor, RepeatedChatMessage, StreamAnswer, StreamComplete, + ChatCloudService, ChatMessage, LocalAIConfig, MessageCursor, RepeatedChatMessage, StreamAnswer, + StreamComplete, }; use flowy_database_pub::cloud::{ CollabDocStateByOid, DatabaseCloudService, DatabaseSnapshot, SummaryRowContent, @@ -727,6 +729,27 @@ impl ChatCloudService for ServerProvider { .stream_complete(&workspace_id, &text, complete_type) .await } + + async fn index_file( + &self, + workspace_id: &str, + file_path: PathBuf, + chat_id: &str, + ) -> Result<(), FlowyError> { + self + .get_server()? + .chat_service() + .index_file(workspace_id, file_path, chat_id) + .await + } + + async fn get_local_ai_config(&self, workspace_id: &str) -> Result { + self + .get_server()? + .chat_service() + .get_local_ai_config(workspace_id) + .await + } } #[async_trait] diff --git a/frontend/rust-lib/flowy-core/src/lib.rs b/frontend/rust-lib/flowy-core/src/lib.rs index 624f423493..ee82157d58 100644 --- a/frontend/rust-lib/flowy-core/src/lib.rs +++ b/frontend/rust-lib/flowy-core/src/lib.rs @@ -86,11 +86,13 @@ impl AppFlowyCore { init_log(&config, &platform, stream_log_sender); } - info!( - "💡{:?}, platform: {:?}", - System::long_os_version(), - platform - ); + if sysinfo::IS_SUPPORTED_SYSTEM { + info!( + "💡{:?}, platform: {:?}", + System::long_os_version(), + platform + ); + } Self::init(config, runtime).await } diff --git a/frontend/rust-lib/flowy-error/Cargo.toml b/frontend/rust-lib/flowy-error/Cargo.toml index cb3864c93c..2eda43b392 100644 --- a/frontend/rust-lib/flowy-error/Cargo.toml +++ b/frontend/rust-lib/flowy-error/Cargo.toml @@ -32,7 +32,7 @@ collab-document = { workspace = true, optional = true } collab-plugins = { workspace = true, optional = true } collab-folder = { workspace = true, optional = true } client-api = { workspace = true, optional = true } -tantivy = { version = "0.21.1", optional = true } +tantivy = { version = "0.22.0", optional = true } [features] impl_from_dispatch_error = ["lib-dispatch"] diff --git a/frontend/rust-lib/flowy-error/src/errors.rs b/frontend/rust-lib/flowy-error/src/errors.rs index e7c6f5a9a6..a442a224ca 100644 --- a/frontend/rust-lib/flowy-error/src/errors.rs +++ b/frontend/rust-lib/flowy-error/src/errors.rs @@ -1,5 +1,5 @@ use std::convert::TryInto; -use std::fmt::Debug; +use std::fmt::{Debug, Display}; use protobuf::ProtobufError; use thiserror::Error; @@ -42,8 +42,8 @@ impl FlowyError { payload: vec![], } } - pub fn with_context(mut self, error: T) -> Self { - self.msg = format!("{:?}", error); + pub fn with_context(mut self, error: T) -> Self { + self.msg = format!("{}", error); self } @@ -137,7 +137,7 @@ pub fn internal_error(e: T) -> FlowyError where T: std::fmt::Debug, { - FlowyError::internal().with_context(e) + FlowyError::internal().with_context(format!("{:?}", e)) } impl std::convert::From for FlowyError { diff --git a/frontend/rust-lib/flowy-search/Cargo.toml b/frontend/rust-lib/flowy-search/Cargo.toml index dbd2b3ecf1..a98ab15a38 100644 --- a/frontend/rust-lib/flowy-search/Cargo.toml +++ b/frontend/rust-lib/flowy-search/Cargo.toml @@ -36,7 +36,7 @@ tracing.workspace = true async-stream = "0.3.4" strsim = "0.11.0" strum_macros = "0.26.1" -tantivy = { version = "0.21.1" } +tantivy = { version = "0.22.0" } tempfile = "3.9.0" validator = { version = "0.16.0", features = ["derive"] } diff --git a/frontend/rust-lib/flowy-search/src/folder/indexer.rs b/frontend/rust-lib/flowy-search/src/folder/indexer.rs index 5831e0871a..6624067788 100644 --- a/frontend/rust-lib/flowy-search/src/folder/indexer.rs +++ b/frontend/rust-lib/flowy-search/src/folder/indexer.rs @@ -22,8 +22,8 @@ use flowy_user::services::authenticate_user::AuthenticateUser; use lib_dispatch::prelude::af_spawn; use strsim::levenshtein; use tantivy::{ - collector::TopDocs, directory::MmapDirectory, doc, query::QueryParser, schema::Field, Index, - IndexReader, IndexWriter, Term, + collector::TopDocs, directory::MmapDirectory, doc, query::QueryParser, schema::Field, Document, + Index, IndexReader, IndexWriter, TantivyDocument, Term, }; use super::entities::FolderIndexData; @@ -233,10 +233,10 @@ impl FolderIndexManagerImpl { let mut search_results: Vec = vec![]; let top_docs = searcher.search(&built_query, &TopDocs::with_limit(10))?; for (_score, doc_address) in top_docs { - let retrieved_doc = searcher.doc(doc_address)?; + let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?; let mut content = HashMap::new(); - let named_doc = folder_schema.schema.to_named_doc(&retrieved_doc); + let named_doc = retrieved_doc.to_named_doc(&folder_schema.schema); for (k, v) in named_doc.0 { content.insert(k, v[0].clone()); } diff --git a/frontend/rust-lib/flowy-search/tests/tantivy_test.rs b/frontend/rust-lib/flowy-search/tests/tantivy_test.rs index b07853c7de..f68b06d610 100644 --- a/frontend/rust-lib/flowy-search/tests/tantivy_test.rs +++ b/frontend/rust-lib/flowy-search/tests/tantivy_test.rs @@ -47,7 +47,7 @@ fn search_folder_test() { for (_score, doc_address) in top_docs { // Retrieve the actual content of documents given its `doc_address`. - let retrieved_doc = searcher.doc(doc_address).unwrap(); - println!("{}", schema.to_json(&retrieved_doc)); + let retrieved_doc: TantivyDocument = searcher.doc(doc_address).unwrap(); + println!("{}", retrieved_doc.to_json(&schema)); } } diff --git a/frontend/rust-lib/flowy-server/src/af_cloud/impls/chat.rs b/frontend/rust-lib/flowy-server/src/af_cloud/impls/chat.rs index d07f53a4a8..e69201a390 100644 --- a/frontend/rust-lib/flowy-server/src/af_cloud/impls/chat.rs +++ b/frontend/rust-lib/flowy-server/src/af_cloud/impls/chat.rs @@ -5,12 +5,14 @@ use client_api::entity::{ RepeatedChatMessage, }; use flowy_chat_pub::cloud::{ - ChatCloudService, ChatMessage, ChatMessageType, StreamAnswer, StreamComplete, + ChatCloudService, ChatMessage, ChatMessageType, LocalAIConfig, StreamAnswer, StreamComplete, }; use flowy_error::FlowyError; use futures_util::{StreamExt, TryStreamExt}; use lib_infra::async_trait::async_trait; use lib_infra::future::FutureResult; +use lib_infra::util::{get_operating_system, OperatingSystem}; +use std::path::PathBuf; pub(crate) struct AFCloudChatCloudServiceImpl { pub inner: T, @@ -182,4 +184,35 @@ where .map_err(FlowyError::from)?; Ok(stream.boxed()) } + + async fn index_file( + &self, + _workspace_id: &str, + _file_path: PathBuf, + _chat_id: &str, + ) -> Result<(), FlowyError> { + return Err( + FlowyError::not_support() + .with_context("indexing file with appflowy cloud is not suppotred yet"), + ); + } + + async fn get_local_ai_config(&self, workspace_id: &str) -> Result { + let system = get_operating_system(); + let platform = match system { + OperatingSystem::MacOS => "macos", + _ => { + return Err( + FlowyError::not_support() + .with_context("local ai is not supported on this operating system"), + ); + }, + }; + let config = self + .inner + .try_get_client()? + .get_local_ai_config(workspace_id, platform) + .await?; + Ok(config) + } } diff --git a/frontend/rust-lib/flowy-server/src/af_cloud/impls/user/cloud_service_impl.rs b/frontend/rust-lib/flowy-server/src/af_cloud/impls/user/cloud_service_impl.rs index f29dc52934..4cd25af7d6 100644 --- a/frontend/rust-lib/flowy-server/src/af_cloud/impls/user/cloud_service_impl.rs +++ b/frontend/rust-lib/flowy-server/src/af_cloud/impls/user/cloud_service_impl.rs @@ -543,7 +543,9 @@ where let try_get_client = self.server.try_get_client(); FutureResult::new(async move { let client = try_get_client?; - client.cancel_subscription(&workspace_id, &SubscriptionPlan::Pro).await?; // TODO: + client + .cancel_subscription(&workspace_id, &SubscriptionPlan::Pro) + .await?; // TODO: Ok(()) }) } @@ -725,11 +727,7 @@ fn to_workspace_subscription_plan( fn to_workspace_subscription(s: WorkspaceSubscriptionStatus) -> WorkspaceSubscription { WorkspaceSubscription { workspace_id: s.workspace_id, - subscription_plan: match s.workspace_plan { - // WorkspaceSubscriptionPlan::Pro => flowy_user_pub::entities::SubscriptionPlan::Pro, - // WorkspaceSubscriptionPlan::Team => flowy_user_pub::entities::SubscriptionPlan::Team, - _ => flowy_user_pub::entities::SubscriptionPlan::None, - }, + subscription_plan: flowy_user_pub::entities::SubscriptionPlan::None, recurring_interval: match s.recurring_interval { client_api::entity::billing_dto::RecurringInterval::Month => { flowy_user_pub::entities::RecurringInterval::Month diff --git a/frontend/rust-lib/flowy-server/src/default_impl.rs b/frontend/rust-lib/flowy-server/src/default_impl.rs index d40100cc15..955cf653da 100644 --- a/frontend/rust-lib/flowy-server/src/default_impl.rs +++ b/frontend/rust-lib/flowy-server/src/default_impl.rs @@ -1,9 +1,10 @@ -use client_api::entity::ai_dto::{CompletionType, RepeatedRelatedQuestion}; +use client_api::entity::ai_dto::{CompletionType, LocalAIConfig, RepeatedRelatedQuestion}; use client_api::entity::{ChatMessageType, MessageCursor, RepeatedChatMessage}; use flowy_chat_pub::cloud::{ChatCloudService, ChatMessage, StreamAnswer, StreamComplete}; use flowy_error::FlowyError; use lib_infra::async_trait::async_trait; use lib_infra::future::FutureResult; +use std::path::PathBuf; pub(crate) struct DefaultChatCloudServiceImpl; @@ -93,4 +94,20 @@ impl ChatCloudService for DefaultChatCloudServiceImpl { ) -> Result { Err(FlowyError::not_support().with_context("complete text is not supported in local server.")) } + + async fn index_file( + &self, + _workspace_id: &str, + _file_path: PathBuf, + _chat_id: &str, + ) -> Result<(), FlowyError> { + Err(FlowyError::not_support().with_context("indexing file is not supported in local server.")) + } + + async fn get_local_ai_config(&self, _workspace_id: &str) -> Result { + Err( + FlowyError::not_support() + .with_context("Get local ai config is not supported in local server."), + ) + } } diff --git a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs index 420ffdeda6..0065bddd14 100644 --- a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs +++ b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs @@ -88,6 +88,11 @@ impl AuthenticateUser { PathBuf::from(self.user_paths.user_data_dir(uid)).join("indexes") } + pub fn get_user_data_dir(&self) -> FlowyResult { + let uid = self.user_id()?; + Ok(PathBuf::from(self.user_paths.user_data_dir(uid))) + } + pub fn get_application_root_dir(&self) -> &str { self.user_paths.root() } diff --git a/frontend/rust-lib/lib-infra/src/file_util.rs b/frontend/rust-lib/lib-infra/src/file_util.rs index 2186c71eaa..c8a464d5dd 100644 --- a/frontend/rust-lib/lib-infra/src/file_util.rs +++ b/frontend/rust-lib/lib-infra/src/file_util.rs @@ -1,10 +1,10 @@ use anyhow::Context; use std::cmp::Ordering; use std::fs::File; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::time::SystemTime; use std::{fs, io}; - use tempfile::tempdir; use walkdir::WalkDir; use zip::write::FileOptions; @@ -69,11 +69,13 @@ where } pub fn zip_folder(src_path: impl AsRef, dest_path: &Path) -> io::Result<()> { - if !src_path.as_ref().exists() { + let src_path = src_path.as_ref(); + + if !src_path.exists() { return Err(io::ErrorKind::NotFound.into()); } - if src_path.as_ref() == dest_path { + if src_path == dest_path { return Err(io::ErrorKind::InvalidInput.into()); } @@ -81,36 +83,39 @@ pub fn zip_folder(src_path: impl AsRef, dest_path: &Path) -> io::Result<() let mut zip = ZipWriter::new(file); let options = FileOptions::default().compression_method(zip::CompressionMethod::Deflated); - for entry in WalkDir::new(&src_path) { + for entry in WalkDir::new(src_path) { let entry = entry?; let path = entry.path(); - let name = match path.strip_prefix(&src_path) { + let name = match path.strip_prefix(src_path) { Ok(n) => n, Err(_) => return Err(io::Error::new(io::ErrorKind::Other, "Invalid path")), }; if path.is_file() { - zip.start_file( - name - .to_str() - .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Invalid file name"))?, - options, - )?; - let mut f = File::open(path)?; - io::copy(&mut f, &mut zip)?; + let file_name = name + .to_str() + .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Invalid file name"))?; + zip.start_file(file_name, options)?; + + let mut buffer = Vec::new(); + { + let mut f = File::open(path)?; + f.read_to_end(&mut buffer)?; + drop(f); + } + + zip.write_all(&buffer)?; } else if !name.as_os_str().is_empty() { - zip.add_directory( - name - .to_str() - .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Invalid directory name"))?, - options, - )?; + let dir_name = name + .to_str() + .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Invalid directory name"))?; + zip.add_directory(dir_name, options)?; } } + zip.finish()?; Ok(()) } - pub fn unzip_and_replace( zip_path: impl AsRef, target_folder: &Path, diff --git a/frontend/rust-lib/lib-infra/src/isolate_stream.rs b/frontend/rust-lib/lib-infra/src/isolate_stream.rs index 19f692dda4..3f2be5477b 100644 --- a/frontend/rust-lib/lib-infra/src/isolate_stream.rs +++ b/frontend/rust-lib/lib-infra/src/isolate_stream.rs @@ -1,4 +1,5 @@ use allo_isolate::{IntoDart, Isolate}; +use anyhow::anyhow; use futures::Sink; use pin_project::pin_project; use std::pin::Pin; @@ -19,7 +20,7 @@ impl Sink for IsolateSink where T: IntoDart, { - type Error = (); + type Error = anyhow::Error; fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) @@ -30,7 +31,7 @@ where if this.isolate.post(item) { Ok(()) } else { - Err(()) + Err(anyhow!("failed to post message")) } }