Spaces:
Sleeping
Sleeping
from langchain_community.embeddings import HuggingFaceEmbeddings | |
from loader import load_and_split_markdown | |
# from langchain.vectorstores import FAISS | |
from langchain_community.vectorstores import FAISS | |
from langchain_community.vectorstores.utils import DistanceStrategy | |
from huggingface_hub.utils import disable_progress_bars | |
disable_progress_bars() # Отключает прогресс-бары загрузки | |
def get_retriever(name='intfloat/multilingual-e5-large'): | |
# Убираем multi_process для Windows | |
embedding_model = HuggingFaceEmbeddings( | |
model_name=name, | |
model_kwargs={"device": "cpu"}, | |
encode_kwargs={ | |
"normalize_embeddings": True, | |
"batch_size": 4 # Уменьшаем batch_size для CPU | |
} | |
) | |
docs_processed=load_and_split_markdown() | |
KNOWLEDGE_VECTOR_DATABASE = FAISS.from_documents( | |
documents=docs_processed, embedding=embedding_model, distance_strategy=DistanceStrategy.COSINE ) | |
return embedding_model, KNOWLEDGE_VECTOR_DATABASE |