Vector.
This commit is contained in:
@@ -1,29 +1,25 @@
|
||||
# app/agents/vectorization_agent.py
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
|
||||
import chromadb
|
||||
|
||||
client = chromadb.PersistentClient(path="./chroma_db")
|
||||
vector_store = client.get_or_create_collection(name="documents")
|
||||
|
||||
text_splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=1000,
|
||||
chunk_overlap=100,
|
||||
)
|
||||
|
||||
def agent_vectorize_and_store(doc_id: str, text: str, category: str, language: str):
|
||||
"""
|
||||
Agent 4: Vectorizes a document and stores it in ChromaDB.
|
||||
"""
|
||||
def agent_vectorize_and_store(
|
||||
doc_id: str,
|
||||
text: str,
|
||||
category: str,
|
||||
language: str,
|
||||
embedding_model,
|
||||
vector_store
|
||||
):
|
||||
print(f"--- [Background Task] Starting vectorization (ID: {doc_id})...")
|
||||
|
||||
try:
|
||||
return
|
||||
|
||||
chunks = text_splitter.split_text(text)
|
||||
if not chunks:
|
||||
print(f"--- [Background Task] document {doc_id} has no text to vectorize.")
|
||||
print(f"--- [Background task] document is empty, skip vectorization. (ID: {doc_id})")
|
||||
return
|
||||
|
||||
chunk_ids = [f"{doc_id}_{i}" for i in range(len(chunks))]
|
||||
@@ -38,6 +34,6 @@ def agent_vectorize_and_store(doc_id: str, text: str, category: str, language: s
|
||||
documents=chunks,
|
||||
metadatas=metadatas
|
||||
)
|
||||
print(f"--- [Background Task] Document {doc_id} vectorized and stored successfully.")
|
||||
print(f"--- [Background Task] Document {doc_id} vectorized。")
|
||||
except Exception as e:
|
||||
print(f"--- [background Task] Vectorization failed (ID: {doc_id}): {e}")
|
||||
print(f"--- [Background Task] Document vectorization failed (ID: {doc_id}): {e}")
|
||||
|
||||
Reference in New Issue
Block a user