Update custom_llm.py

custom_llm.py

@@ -31,14 +31,14 @@ import pickle, asyncio
 async def create_vectorstore():
     API_TOKEN = os.getenv('HF_INFER_API')
     
-    loader = os.getenv('knowledge_base')
+    # loader = os.getenv('knowledge_base')
     # web_loader = load_web("https://lintasmediadanawa.com")
     
-    splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=20)
+    # splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=20)
     
     # docs = splitter.create_documents([loader]+web_loader)
-    docs = splitter.create_documents([loader])
-    print(len(docs))
+    # docs = splitter.create_documents([loader])
+    # print(len(docs))
     emb_model = HuggingFaceEmbeddings(model_name='sentence-transformers/paraphrase-multilingual-mpnet-base-v2', encode_kwargs={'normalize_embeddings': True})
 
     # emb_model = HuggingFaceInferenceAPIEmbeddings(
@@ -48,18 +48,13 @@ async def create_vectorstore():
     async def add_docs(d):
         db.aadd_documents(await splitter.atransform_documents([d]))
         
-    db = await FAISS.afrom_documents(docs, emb_model)
+    # db = FAISS.afrom_documents(docs, emb_model)
 
     f = pickle.load(open("ebesha_ticket.pkl", "rb"))
 
     print("Docs len :", len(f))
 
-    tasks = []
-
-    for d in f:
-        tasks.append(db.aadd_documents(await splitter.atransform_documents([d])))
-
-    await asyncio.gather(*tasks)
+    db = FAISS.from_documents(f, emb_model)