From f7adcae8ffd45175017c1d90c1a0961df93d64b9 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 12 Aug 2024 09:21:44 +0800 Subject: [PATCH] chore: show attachment on local ai (#5929) * chore: show attachment on local ai * chore: fix compile --- .../application/chat_ai_message_bloc.dart | 4 +-- .../ai_chat/application/chat_bloc.dart | 27 ++++++++++++----- .../ai_chat/application/chat_entity.dart | 10 +++++-- .../application/chat_message_service.dart | 29 ++++++++++++++----- .../application/chat_message_stream.dart | 2 +- .../chat_user_message_bubble_bloc.dart | 16 ++++++++-- .../lib/plugins/ai_chat/chat_page.dart | 7 ++--- .../presentation/message/ai_text_message.dart | 6 ++-- .../message/user_message_bubble.dart | 2 -- .../message/user_text_message.dart | 2 -- .../src/middleware/chat_service_mw.rs | 1 + 11 files changed, 74 insertions(+), 32 deletions(-) diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart index 9083e40521..19fff60bbf 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart @@ -16,13 +16,13 @@ part 'chat_ai_message_bloc.freezed.dart'; class ChatAIMessageBloc extends Bloc { ChatAIMessageBloc({ dynamic message, - String? metadata, + String? refSourceJsonString, required this.chatId, required this.questionId, }) : super( ChatAIMessageState.initial( message, - messageRefSourceFromString(metadata), + messageReferenceSource(refSourceJsonString), ), ) { if (state.stream != null) { diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart index cd5579d7d4..09bf21acaa 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart @@ -374,7 +374,10 @@ class ChatBloc extends Bloc { metadata: await metadataPBFromMetadata(metadata), ); - final questionStreamMessage = _createQuestionStreamMessage(questionStream); + final questionStreamMessage = _createQuestionStreamMessage( + questionStream, + metadata, + ); add(ChatEvent.receveMessage(questionStreamMessage)); // Stream message to the server @@ -432,14 +435,24 @@ class ChatBloc extends Bloc { ); } - Message _createQuestionStreamMessage(QuestionStream stream) { + Message _createQuestionStreamMessage( + QuestionStream stream, + Map? sentMetadata, + ) { questionStreamMessageId = nanoid(); + final Map metadata = {}; + + // if (sentMetadata != null) { + // metadata[messageMetadataJsonStringKey] = sentMetadata; + // } + + metadata["$QuestionStream"] = stream; + metadata["chatId"] = chatId; + metadata[messageChatFileListKey] = + chatFilesFromMessageMetadata(sentMetadata); return TextMessage( author: User(id: state.userProfile.id.toString()), - metadata: { - "$QuestionStream": stream, - "chatId": chatId, - }, + metadata: metadata, id: questionStreamMessageId, createdAt: DateTime.now().millisecondsSinceEpoch, text: '', @@ -460,7 +473,7 @@ class ChatBloc extends Bloc { text: message.content, createdAt: message.createdAt.toInt() * 1000, metadata: { - messageMetadataKey: message.metadata, + messageRefSourceJsonStringKey: message.metadata, }, ); } diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart index 99b6b66bb1..c0a2d5bf32 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart @@ -15,8 +15,14 @@ part 'chat_entity.freezed.dart'; const sendMessageErrorKey = "sendMessageError"; const systemUserId = "system"; const aiResponseUserId = "0"; -const messageMetadataKey = "metadata"; -const messageQuestionIdKey = "question"; + +/// `messageRefSourceJsonStringKey` is the key used for metadata that contains the reference source of a message. +/// Each message may include this information. +/// - When used in a sent message, it indicates that the message includes an attachment. +/// - When used in a received message, it indicates the AI reference sources used to answer a question. +const messageRefSourceJsonStringKey = "ref_source_json_string"; +const messageChatFileListKey = "chat_files"; +const messageQuestionIdKey = "question_id"; @JsonSerializable() class ChatMessageRefSource { diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart index a2cfc9f986..c497cd6c23 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart @@ -32,7 +32,12 @@ List chatFilesFromMetadataString(String? s) { final metadataJson = jsonDecode(s); if (metadataJson is Map) { - return _parseChatFile(metadataJson); + final file = chatFileFromMap(metadataJson); + if (file != null) { + return [file]; + } else { + return []; + } } else if (metadataJson is List) { return metadataJson .map((e) => e as Map) @@ -46,11 +51,6 @@ List chatFilesFromMetadataString(String? s) { } } -List _parseChatFile(Map map) { - final file = chatFileFromMap(map); - return file != null ? [file] : []; -} - ChatFile? chatFileFromMap(Map? map) { if (map == null) return null; @@ -63,7 +63,7 @@ ChatFile? chatFileFromMap(Map? map) { return ChatFile.fromFilePath(filePath); } -List messageRefSourceFromString(String? s) { +List messageReferenceSource(String? s) { if (s == null || s.isEmpty || s == "null") { return []; } @@ -139,3 +139,18 @@ Future> metadataPBFromMetadata( return metadata; } + +List chatFilesFromMessageMetadata( + Map? map, +) { + final List metadata = []; + if (map != null) { + for (final entry in map.entries) { + if (entry.value is ChatFile) { + metadata.add(entry.value); + } + } + } + + return metadata; +} diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart index a82a84f34a..438f3874e2 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart @@ -25,7 +25,7 @@ class AnswerStream { } else if (event.startsWith("metadata:")) { if (_onMetadata != null) { final s = event.substring(9); - _onMetadata!(messageRefSourceFromString(s)); + _onMetadata!(messageReferenceSource(s)); } } else if (event == "AI_RESPONSE_LIMIT") { if (_onAIResponseLimit != null) { diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart index 542280ca99..c4571915df 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart @@ -11,11 +11,10 @@ class ChatUserMessageBubbleBloc extends Bloc { ChatUserMessageBubbleBloc({ required Message message, - required String? metadata, }) : super( ChatUserMessageBubbleState.initial( message, - chatFilesFromMetadataString(metadata), + _getFiles(message.metadata), ), ) { on( @@ -28,6 +27,19 @@ class ChatUserMessageBubbleBloc } } +List _getFiles(Map? metadata) { + if (metadata == null) { + return []; + } + final refSourceMetadata = metadata[messageRefSourceJsonStringKey] as String?; + final files = metadata[messageChatFileListKey] as List?; + + if (refSourceMetadata != null) { + return chatFilesFromMetadataString(refSourceMetadata); + } + return files ?? []; +} + @freezed class ChatUserMessageBubbleEvent with _$ChatUserMessageBubbleEvent { const factory ChatUserMessageBubbleEvent.initial() = Initial; diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart index 6ad35d08db..fced505f20 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart @@ -320,17 +320,16 @@ class _ChatContentPageState extends State<_ChatContentPage> { Widget _buildTextMessage(BuildContext context, TextMessage message) { if (message.author.id == _user.id) { final stream = message.metadata?["$QuestionStream"]; - final metadata = message.metadata?[messageMetadataKey] as String?; return ChatUserMessageWidget( key: ValueKey(message.id), user: message.author, message: stream is QuestionStream ? stream : message.text, - metadata: metadata, ); } else { final stream = message.metadata?["$AnswerStream"]; final questionId = message.metadata?[messageQuestionIdKey]; - final metadata = message.metadata?[messageMetadataKey] as String?; + final refSourceJsonString = + message.metadata?[messageRefSourceJsonStringKey] as String?; return ChatAIMessageWidget( user: message.author, messageUserId: message.id, @@ -338,7 +337,7 @@ class _ChatContentPageState extends State<_ChatContentPage> { key: ValueKey(message.id), questionId: questionId, chatId: widget.view.id, - metadata: metadata, + refSourceJsonString: refSourceJsonString, onSelectedMetadata: (ChatMessageRefSource metadata) { context.read().add( ChatSidePannelEvent.selectedMetadata(metadata), diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart index ed928eba14..7aa48b094f 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart @@ -22,7 +22,7 @@ class ChatAIMessageWidget extends StatelessWidget { required this.message, required this.questionId, required this.chatId, - required this.metadata, + required this.refSourceJsonString, required this.onSelectedMetadata, }); @@ -33,7 +33,7 @@ class ChatAIMessageWidget extends StatelessWidget { final dynamic message; final Int64? questionId; final String chatId; - final String? metadata; + final String? refSourceJsonString; final void Function(ChatMessageRefSource metadata) onSelectedMetadata; @override @@ -41,7 +41,7 @@ class ChatAIMessageWidget extends StatelessWidget { return BlocProvider( create: (context) => ChatAIMessageBloc( message: message, - metadata: metadata, + refSourceJsonString: refSourceJsonString, chatId: chatId, questionId: questionId, )..add(const ChatAIMessageEvent.initial()), diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart index 06401b76a7..08c627b0b7 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart @@ -29,12 +29,10 @@ class ChatUserMessageBubble extends StatelessWidget { .read() .add(ChatMemberEvent.getMemberInfo(message.author.id)); } - final metadata = message.metadata?[messageMetadataKey] as String?; return BlocProvider( create: (context) => ChatUserMessageBubbleBloc( message: message, - metadata: metadata, ), child: BlocBuilder( builder: (context, state) { diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart index 0b2a2efa7d..c804441188 100644 --- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart +++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart @@ -10,12 +10,10 @@ class ChatUserMessageWidget extends StatelessWidget { super.key, required this.user, required this.message, - required this.metadata, }); final User user; final dynamic message; - final String? metadata; @override Widget build(BuildContext context) { diff --git a/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs b/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs index 8da2505e75..2e7f4e54a2 100644 --- a/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs +++ b/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs @@ -63,6 +63,7 @@ impl AICloudServiceMiddleware { let _ = index_process_sink .send(StreamMessage::IndexStart.to_string()) .await; + self .local_llm_controller .index_message_metadata(chat_id, metadata_list, index_process_sink)