
from langchain_community.document_loaders import PyPDFLoader
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.document_loaders import HNLoader
from langchain.text_splitter import CharacterTextSplitter
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import UnstructuredHTMLLoader
from langchain_openai.embeddings import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
from langchain.chains import RetrievalQA
from langchain.chains import RetrievalQAWithSourcesChain
from langchain_openai.llms import OpenAI

from constant import openai
import os

os.environ['OPENAI_API_KEY'] = openai

loader = PyPDFLoader("attention is all you need.pdf")
data = loader.load()
# print(data[0])

loader = CSVLoader(file_path="job_placement.csv")
data = loader.load()
# print(data[0])

loader = HNLoader("https://news.ycombinator.com")
data = loader.load()
# print(data[0])


quote = "one Machine can do the work of fifty ordinary humans, No machine can do the" \
        "work of one extraordinary human."

ct_splitter = CharacterTextSplitter(
    separator='.',
    chunk_size=24,
    chunk_overlap=3
)

# docs = ct_splitter.split_text(quote)
# print(docs)

rc_splitter = RecursiveCharacterTextSplitter(
    chunk_size=24,
    chunk_overlap=3,
)

# docs = rc_splitter.split_text(quote)
# print(docs)

loader = UnstructuredHTMLLoader("data.html")
data = loader.load()

rc_splitter = RecursiveCharacterTextSplitter(
    chunk_size=24,
    chunk_overlap=3,
    separators='.',
)

# docs = rc_splitter.split_documents(data)
# print(docs)

quote = "There is a kingdom of lychee fruit that are alive and thriving in Iceland, but they feel " \
        "taken advantage of and are not fast enough for you."

splitter = RecursiveCharacterTextSplitter(
    chunk_size=40,
    chunk_overlap=10,
)

docs = splitter.split_text(quote)

embeddings = OpenAIEmbeddings(openai_api_key=openai)

vectordb = Chroma(
    persist_directory="data",
    embedding_function=embeddings
)

vectordb.persist()

docstorage = Chroma.from_texts(docs,embeddings)

qa = RetrievalQA.from_chain_type(
    llm = OpenAI(model_name="gpt-3.5-turbo-instruct"),
    chain_type="stuff",
    retriever = docstorage.as_retriever()
)

# query = "Where do lychee fruit live?"
# print(qa.invoke(query))

quote = "There is a kingdom of lycee fruit that are alive and thriving in Iceland, but they fee" \
        "taken advantage of and are not fast enough for you."

qa1 = RetrievalQAWithSourcesChain.from_chain_type(
    llm = OpenAI(model_name="gpt-3.5-turbo-instruct"),
    chain_type="stuff",
    retriever = docstorage.as_retriever(),
)

results = qa1({'question':'What is the primary architecture presented in the document?'},return_only_outputs=True)
print(results)

langchain_community.document_loaders.PyPDFLoader langchain.text_splitter.CharacterTextSplitter langchain_openai.llms.OpenAI langchain_community.document_loaders.csv_loader.CSVLoader langchain.text_splitter.RecursiveCharacterTextSplitter langchain_community.document_loaders.UnstructuredHTMLLoader langchain_community.document_loaders.HNLoader langchain_community.vectorstores.Chroma.from_texts langchain_community.vectorstores.Chroma langchain_openai.embeddings.OpenAIEmbeddings
[((741, 785), 'langchain_community.document_loaders.PyPDFLoader', 'PyPDFLoader', (['"""attention is all you need.pdf"""'], {}), "('attention is all you need.pdf')\n", (752, 785), False, 'from langchain_community.document_loaders import PyPDFLoader\n'), ((838, 878), 'langchain_community.document_loaders.csv_loader.CSVLoader', 'CSVLoader', ([], {'file_path': '"""job_placement.csv"""'}), "(file_path='job_placement.csv')\n", (847, 878), False, 'from langchain_community.document_loaders.csv_loader import CSVLoader\n'), ((931, 971), 'langchain_community.document_loaders.HNLoader', 'HNLoader', (['"""https://news.ycombinator.com"""'], {}), "('https://news.ycombinator.com')\n", (939, 971), False, 'from langchain_community.document_loaders import HNLoader\n'), ((1166, 1234), 'langchain.text_splitter.CharacterTextSplitter', 'CharacterTextSplitter', ([], {'separator': '"""."""', 'chunk_size': '(24)', 'chunk_overlap': '(3)'}), "(separator='.', chunk_size=24, chunk_overlap=3)\n", (1187, 1234), False, 'from langchain.text_splitter import CharacterTextSplitter\n'), ((1327, 1389), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(24)', 'chunk_overlap': '(3)'}), '(chunk_size=24, chunk_overlap=3)\n', (1357, 1389), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1473, 1508), 'langchain_community.document_loaders.UnstructuredHTMLLoader', 'UnstructuredHTMLLoader', (['"""data.html"""'], {}), "('data.html')\n", (1495, 1508), False, 'from langchain_community.document_loaders import UnstructuredHTMLLoader\n'), ((1548, 1626), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(24)', 'chunk_overlap': '(3)', 'separators': '"""."""'}), "(chunk_size=24, chunk_overlap=3, separators='.')\n", (1578, 1626), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1889, 1952), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(40)', 'chunk_overlap': '(10)'}), '(chunk_size=40, chunk_overlap=10)\n', (1919, 1952), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((2020, 2059), 'langchain_openai.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {'openai_api_key': 'openai'}), '(openai_api_key=openai)\n', (2036, 2059), False, 'from langchain_openai.embeddings import OpenAIEmbeddings\n'), ((2074, 2137), 'langchain_community.vectorstores.Chroma', 'Chroma', ([], {'persist_directory': '"""data"""', 'embedding_function': 'embeddings'}), "(persist_directory='data', embedding_function=embeddings)\n", (2080, 2137), False, 'from langchain_community.vectorstores import Chroma\n'), ((2189, 2224), 'langchain_community.vectorstores.Chroma.from_texts', 'Chroma.from_texts', (['docs', 'embeddings'], {}), '(docs, embeddings)\n', (2206, 2224), False, 'from langchain_community.vectorstores import Chroma\n'), ((2272, 2315), 'langchain_openai.llms.OpenAI', 'OpenAI', ([], {'model_name': '"""gpt-3.5-turbo-instruct"""'}), "(model_name='gpt-3.5-turbo-instruct')\n", (2278, 2315), False, 'from langchain_openai.llms import OpenAI\n'), ((2688, 2731), 'langchain_openai.llms.OpenAI', 'OpenAI', ([], {'model_name': '"""gpt-3.5-turbo-instruct"""'}), "(model_name='gpt-3.5-turbo-instruct')\n", (2694, 2731), False, 'from langchain_openai.llms import OpenAI\n')]