import os

from dotenv import load_dotenv, find_dotenv
_ = load_dotenv(find_dotenv()) # read local .env file


from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import CSVLoader
from langchain.indexes import VectorstoreIndexCreator
from langchain.vectorstores import DocArrayInMemorySearch

file = 'OutdoorClothingCatalog_1000.csv'
loader = CSVLoader(file_path=file)
data = loader.load()

index = VectorstoreIndexCreator(
    vectorstore_cls=DocArrayInMemorySearch
).from_loaders([loader])

llm = ChatOpenAI(temperature = 0.0)
qa = RetrievalQA.from_chain_type(
    llm=llm, 
    chain_type="stuff", 
    retriever=index.vectorstore.as_retriever(), 
    verbose=True,
    chain_type_kwargs = {
        "document_separator": "<<<<>>>>>"
    }
)

data[10]

# Takes in document and creates QA pairs for each document
from langchain.evaluation.qa import QAGenerateChain
example_gen_chain = QAGenerateChain.from_llm(ChatOpenAI())

new_examples = example_gen_chain.apply_and_parse(
    [{"doc": t} for t in data[:5]]
)

new_examples[0]

examples = [
    {
        "query": "Do the Cozy Comfort Pullover Set\
        have side pockets?",
        "answer": "Yes"
    },
    {
        "query": "What collection is the Ultra-Lofty \
        850 Stretch Down Hooded Jacket from?",
        "answer": "The DownTek collection"
    }
]

from langchain.evaluation.qa import QAGenerateChain

example_gen_chain = QAGenerateChain.from_llm(ChatOpenAI())
new_examples = example_gen_chain.apply_and_parse(
    [{"doc": t} for t in data[:5]]
)
new_examples[0]
data[0]
examples += new_examples
qa.run(examples[0]["query"])

# Manual evaluation

import langchain
langchain.debug = True

qa.run(examples[0]["query"])

# Turn off the debug mode
langchain.debug = False

predictions = qa.apply(examples)

from langchain.evaluation.qa import QAEvalChain

llm = ChatOpenAI(temperature=0)
eval_chain = QAEvalChain.from_llm(llm)

graded_outputs = eval_chain.evaluate(examples, predictions)

for i, eg in enumerate(examples):
    print(f"Example {i}:")
    print("Question: " + predictions[i]['query'])
    print("Real Answer: " + predictions[i]['answer'])
    print("Predicted Answer: " + predictions[i]['result'])
    print("Predicted Grade: " + graded_outputs[i]['text'])
    print()
langchain.evaluation.qa.QAEvalChain.from_llm langchain.document_loaders.CSVLoader langchain.indexes.VectorstoreIndexCreator langchain.chat_models.ChatOpenAI
[((409, 434), 'langchain.document_loaders.CSVLoader', 'CSVLoader', ([], {'file_path': 'file'}), '(file_path=file)\n', (418, 434), False, 'from langchain.document_loaders import CSVLoader\n'), ((565, 592), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': '(0.0)'}), '(temperature=0.0)\n', (575, 592), False, 'from langchain.chat_models import ChatOpenAI\n'), ((1899, 1924), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': '(0)'}), '(temperature=0)\n', (1909, 1924), False, 'from langchain.chat_models import ChatOpenAI\n'), ((1938, 1963), 'langchain.evaluation.qa.QAEvalChain.from_llm', 'QAEvalChain.from_llm', (['llm'], {}), '(llm)\n', (1958, 1963), False, 'from langchain.evaluation.qa import QAEvalChain\n'), ((71, 84), 'dotenv.find_dotenv', 'find_dotenv', ([], {}), '()\n', (82, 84), False, 'from dotenv import load_dotenv, find_dotenv\n'), ((978, 990), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {}), '()\n', (988, 990), False, 'from langchain.chat_models import ChatOpenAI\n'), ((1487, 1499), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {}), '()\n', (1497, 1499), False, 'from langchain.chat_models import ChatOpenAI\n'), ((465, 528), 'langchain.indexes.VectorstoreIndexCreator', 'VectorstoreIndexCreator', ([], {'vectorstore_cls': 'DocArrayInMemorySearch'}), '(vectorstore_cls=DocArrayInMemorySearch)\n', (488, 528), False, 'from langchain.indexes import VectorstoreIndexCreator\n')]