update

app.py

@@ -7,6 +7,15 @@ import chainlit as cl  # importing chainlit for our app
 from chainlit.prompt import Prompt, PromptMessage  # importing prompt tools
 from chainlit.playground.providers import ChatOpenAI  # importing ChatOpenAI tools
 from dotenv import load_dotenv
+from langchain.document_loaders import CSVLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain.embeddings import CacheBackedEmbeddings
+from langchain.storage import LocalFileStore
+from langchain_community.vectorstores import FAISS
+from datasets import load_dataset
+
+
 
 load_dotenv()
 
@@ -18,6 +27,59 @@ user_template = """{input}
 Think through your response step by step.
 """
 
+def setup():
+    dataset = load_dataset("ShubhamChoksi/IMDB_Movies")
+    print(dataset['train'][0])
+    print("data from huggingface dataset\n")
+    
+    dataset_dict = dataset
+    dataset_dict["train"] # TODO - what method do we have to use to store imdb.csv from ShubhamChoksi/IMDB_Movies?
+
+    dataset_dict["train"].to_csv("imdb.csv")
+    loader = CSVLoader(file_path='imdb.csv')
+    data = loader.load()
+    len(data)
+    print(data[0])
+    print("loaded data from csv\n")
+    
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size = 1000,
+        chunk_overlap = 100,
+    )
+
+    chunked_documents = text_splitter.split_documents(data)
+
+
+    len(chunked_documents) # ensure we have actually split the data into chunks
+    print(chunked_documents[0])
+    
+    openai_api_key =  os.getenv("OPENAI_API_KEY")
+
+    embedding_model = OpenAIEmbeddings(openai_api_key=openai_api_key)
+    
+    
+    store = LocalFileStore("./cache/")
+    embedder = CacheBackedEmbeddings.from_bytes_store(
+        embedding_model, store, namespace=embedding_model.model
+    )
+    
+    vector_store = FAISS.from_documents(chunked_documents, embedder)
+
+    vector_store.save_local("./vector_store")
+    
+    vector_store = FAISS.load_local("./vector_store", embedder, allow_dangerous_deserialization=True)
+
+    retriever = vector_store.as_retriever()
+    
+    query = "What are some good westerns movies?"
+    embedded_query = embedding_model.embed_query(query)
+    similar_documents = vector_store.similarity_search_by_vector(embedded_query)
+    for page in similar_documents:
+        # TODO: Print the similar documents that the similarity search returns?
+        print(page)
+        print("00-----0000")
+        print(page)
+        print("-------------")
 
 @cl.on_chat_start  # marks a function that will be executed at the start of a user session
 async def start_chat():