diff --git a/letta/embeddings.py b/letta/embeddings.py index ead188f715..632719d87d 100644 --- a/letta/embeddings.py +++ b/letta/embeddings.py @@ -3,18 +3,7 @@ from typing import Any, List, Optional import numpy as np - -# from llama_index.core.base.embeddings import BaseEmbedding -# from llama_index.core.embeddings import BaseEmbedding -# from llama_index.core.base.embeddings.base import BaseEmbedding -# from llama_index.bridge.pydantic import PrivateAttr -# from llama_index.embeddings.base import BaseEmbedding -# from llama_index.embeddings.huggingface_utils import format_text import tiktoken -from llama_index.core import Document as LlamaIndexDocument - -# from llama_index.core.base.embeddings import BaseEmbedding -from llama_index.core.node_parser import SentenceSplitter from letta.constants import ( EMBEDDING_TO_TOKENIZER_DEFAULT, @@ -26,6 +15,9 @@ def parse_and_chunk_text(text: str, chunk_size: int) -> List[str]: + from llama_index.core import Document as LlamaIndexDocument + from llama_index.core.node_parser import SentenceSplitter + parser = SentenceSplitter(chunk_size=chunk_size) llama_index_docs = [LlamaIndexDocument(text=text)] nodes = parser.get_nodes_from_documents(llama_index_docs)