diff --git a/src/memos/mem_reader/read_multi_modal/base.py b/src/memos/mem_reader/read_multi_modal/base.py index 95d427864..737a3fe1e 100644 --- a/src/memos/mem_reader/read_multi_modal/base.py +++ b/src/memos/mem_reader/read_multi_modal/base.py @@ -15,6 +15,7 @@ TextualMemoryItem, TreeNodeTextualMemoryMetadata, ) +from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import FastTokenizer from memos.utils import timed from .utils import detect_lang, get_text_splitter @@ -90,6 +91,7 @@ def __init__(self, embedder, llm=None): """ self.embedder = embedder self.llm = llm + self.tokenizer = FastTokenizer(use_jieba=True, use_stopwords=True) @abstractmethod def create_source( diff --git a/src/memos/mem_reader/read_multi_modal/file_content_parser.py b/src/memos/mem_reader/read_multi_modal/file_content_parser.py index 00da08b1c..9f4ab94c2 100644 --- a/src/memos/mem_reader/read_multi_modal/file_content_parser.py +++ b/src/memos/mem_reader/read_multi_modal/file_content_parser.py @@ -710,7 +710,7 @@ def _make_fallback( chunk_idx: int, chunk_text: str, reason: str = "raw" ) -> TextualMemoryItem: """Create fallback memory item with raw chunk text.""" - return _make_memory_item( + raw_chunk_mem = _make_memory_item( value=chunk_text, tags=[ "mode:fine", @@ -721,6 +721,11 @@ def _make_fallback( chunk_idx=chunk_idx, chunk_content=chunk_text, ) + tags_list = self.tokenizer.tokenize_mixed(raw_chunk_mem.metadata.key) + tags_list = [tag for tag in tags_list if len(tag) > 1] + tags_list = sorted(tags_list, key=len, reverse=True) + raw_chunk_mem.metadata.tags.extend(tags_list[:5]) + return raw_chunk_mem # Handle empty chunks case if not valid_chunks: