added index for lib_serach

.gitattributes

@@ -45,3 +45,15 @@ data/smu_lib_index/2.pdf filter=lfs diff=lfs merge=lfs -text
 data/smu_lib_index/3.pdf filter=lfs diff=lfs merge=lfs -text
 data/smu_lib_index/4.pdf filter=lfs diff=lfs merge=lfs -text
 data/smu_lib_index/5.pdf filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/1.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/1.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/4.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/0.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/2.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/2.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/3.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/3.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/4.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/5.pdf/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/5.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/smu_lib_index/0.pdf/index.pkl filter=lfs diff=lfs merge=lfs -text

app_modules/init.py

@@ -26,6 +26,26 @@ llm_loader = None
 qa_chain = None
 
 
+def load_vectorstor(using_faiss, index_path, embeddings):
+    start = timer()
+
+    print(f"Load index from {index_path} with {'FAISS' if using_faiss else 'Chroma'}")
+
+    if not os.path.isdir(index_path):
+        raise ValueError(f"{index_path} does not exist!")
+    elif using_faiss:
+        vectorstore = FAISS.load_local(index_path, embeddings)
+    else:
+        vectorstore = Chroma(
+            embedding_function=embeddings, persist_directory=index_path
+        )
+
+    end = timer()
+
+    print(f"Completed in {end - start:.3f}s")
+    return vectorstore
+
+
 def app_init(initQAChain: bool = True):
     global llm_loader
     global qa_chain
@@ -59,24 +79,7 @@ def app_init(initQAChain: bool = True):
 
             print(f"Completed in {end - start:.3f}s")
 
-            start = timer()
-
-            print(
-                f"Load index from {index_path} with {'FAISS' if using_faiss else 'Chroma'}"
-            )
-
-            if not os.path.isdir(index_path):
-                raise ValueError(f"{index_path} does not exist!")
-            elif using_faiss:
-                vectorstore = FAISS.load_local(index_path, embeddings)
-            else:
-                vectorstore = Chroma(
-                    embedding_function=embeddings, persist_directory=index_path
-                )
-
-            end = timer()
-
-            print(f"Completed in {end - start:.3f}s")
+            vectorstore = load_vectorstor(index_path)
 
         start = timer()
         llm_loader = LLMLoader(llm_model_type)

data/smu_lib_index/0.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b3d8640d878840dd03569beb3de62a0e7db3c69e602c4af94e7f2b4aa7437f1
+size 1665069

data/smu_lib_index/0.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7901a5deeb83aa9042c9d530af0dd4a71d8957ab6967c077fb85afb7e99ed1
+size 495066

data/smu_lib_index/1.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e033249266139624933e8caaa3e477aba0ab95246e8185099b8d6424cfd760e3
+size 2623533

data/smu_lib_index/1.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dd94b10a724cd057b022ff1f1cc986138912e87f285b9f20295d0a4c6fa66f8
+size 754144

data/smu_lib_index/2.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:add4ca6be98c0cced1807a6f8857273aa603af709688903082bac8a135f176bb
+size 113709

data/smu_lib_index/2.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00e5b92d42a46ffd810a6700e31ddb6d25e02d634023b6d098172c9580eecb20
+size 35980

data/smu_lib_index/3.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32c40ae8cf525f67b006d0086904bc59620c744ae69f36db90fdd3c4ae01c892
+size 110637

data/smu_lib_index/3.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0209fd9dfe7e368a6554f9acab7b66d1c56f54101ed135b91d001211a551d4a2
+size 35875

data/smu_lib_index/4.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c12431becadaed99d1b5555540cb0a721838dd0d9d52dfb200fda2ee3eaf280
+size 76845

data/smu_lib_index/4.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430b96fb402101066564b1c8693cb92c18b4f1d42bd8c8e6d90d9c374a5a1b3d
+size 24951

data/smu_lib_index/5.pdf/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbd32f3e66f36b4bbedd2bda520375c7ae37704b49fc65746ebba0d793cecde9
+size 119853

data/smu_lib_index/5.pdf/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1143d4ced259bbc09282f12f39e78b266ae06545b7bfb1fd0758df7f20ff0e0
+size 39268

ingest-lib-search.py

@@ -0,0 +1,119 @@
+# setting device on GPU if available, else CPU
+import os
+from timeit import default_timer as timer
+from typing import List
+
+from langchain.document_loaders import DirectoryLoader
+from langchain.document_loaders import PyPDFDirectoryLoader
+
+from langchain.embeddings import HuggingFaceInstructEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores.base import VectorStore
+from langchain.vectorstores.chroma import Chroma
+from langchain.vectorstores.faiss import FAISS
+
+from app_modules.init import *
+
+
+def load_documents(source_path) -> List:
+    loader = PyPDFDirectoryLoader(source_path, silent_errors=True)
+    documents = loader.load()
+
+    loader = DirectoryLoader(
+        source_path, glob="**/*.html", silent_errors=True, show_progress=True
+    )
+    documents.extend(loader.load())
+    return documents
+
+
+def split_chunks(documents: List, chunk_size, chunk_overlap) -> List:
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap
+    )
+    return text_splitter.split_documents(documents)
+
+
+def generate_index(
+    filename: str, chunks: List, embeddings: HuggingFaceInstructEmbeddings
+) -> VectorStore:
+    full_path = index_path + filename + "/"
+    os.mkdir(full_path)
+
+    if using_faiss:
+        faiss_instructor_embeddings = FAISS.from_documents(
+            documents=chunks, embedding=embeddings
+        )
+
+        faiss_instructor_embeddings.save_local(full_path)
+        return faiss_instructor_embeddings
+    else:
+        chromadb_instructor_embeddings = Chroma.from_documents(
+            documents=chunks, embedding=embeddings, persist_directory=full_path
+        )
+
+        chromadb_instructor_embeddings.persist()
+        return chromadb_instructor_embeddings
+
+
+# Constants
+device_type, hf_pipeline_device_type = get_device_types()
+hf_embeddings_model_name = (
+    os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
+)
+index_path = os.environ.get("FAISS_INDEX_PATH_PDFS") or os.environ.get(
+    "CHROMADB_INDEX_PATH_PDFS"
+)
+using_faiss = os.environ.get("FAISS_INDEX_PATH_PDFS") is not None
+source_path = os.environ.get("SOURCE_PDFS_PATH")
+chunk_size = os.environ.get("CHUNCK_SIZE")
+chunk_overlap = os.environ.get("CHUNK_OVERLAP")
+
+start = timer()
+embeddings = HuggingFaceInstructEmbeddings(
+    model_name=hf_embeddings_model_name, model_kwargs={"device": device_type}
+)
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")
+
+start = timer()
+
+if not os.path.isdir(index_path):
+    print(
+        f"The index persist directory {index_path} is not present. Creating a new one."
+    )
+    os.mkdir(index_path)
+
+    print(f"Loading PDF & HTML files from {source_path}")
+    sources = load_documents(source_path)
+    print(sources[2])
+
+    print(f"Splitting {len(sources)} PDF pages in to chunks ...")
+
+    current_file = None
+    docs = []
+    index = 0
+    for index, doc in enumerate(sources):
+        filename = doc.metadata["source"].split("/")[-1]
+        # print(filename)
+        if (
+            filename != current_file
+            and current_file != None
+            or index == len(sources) - 1
+        ):
+            chunks = split_chunks(
+                docs, chunk_size=int(chunk_size), chunk_overlap=int(chunk_overlap)
+            )
+            print(f"Generating index for {current_file} with {len(chunks)} chunks ...")
+            generate_index(current_file, chunks, embeddings)
+            docs = [doc]
+        else:
+            docs.append(doc)
+
+        current_file = filename
+else:
+    print(f"The index persist directory {index_path} is present. Quitting ...")
+
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")