import os
import time
import openai
import pickle
import langchain
import streamlit as st
from langchain import OpenAI
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import UnstructuredURLLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS

os.environ['OPENAI_API_KEY'] = open("API_KEY","r").read()

#user interface using streamlit
st.title("News Research tool ")
st.sidebar.title("News Article URLs")

urls =[]
for i in range(3):
    url = st.sidebar.text_input(f" URL {i+1} ")
    urls.append(url)

process_url_clicked = st.sidebar.button("Process URLs")
file_path = "faiss_store_openai.pkl"

main_placeholder = st.empty()
llm = OpenAI(temperature=0.8, max_tokens=500)

if process_url_clicked:
    # data loader
    loader = UnstructuredURLLoader(urls=urls)
    main_placeholder.text("data loading....started....")
    data = loader.load()
    # splitting the data into chunks
    text_splitter = RecursiveCharacterTextSplitter(
        separators=['\nn', '\n', '.', ' ' ],
        chunk_size=1000
    )
    main_placeholder.text("data splitting ....started....")
    docs = text_splitter.split_documents(data)
    # embedding into vector format
    embeddings = OpenAIEmbeddings()
    vectorstore_openai = FAISS.from_documents(docs,embeddings)
    main_placeholder.text("Embedding Vector....started....")
    time.sleep(2)
    # saving the FAISS file to a pickle file
    with open(file_path, "wb") as f:
        pickle.dump(vectorstore_openai,f)

query = main_placeholder.text_input("Question: ")
if query:
    if os.path.exists(file_path):
        with open(file_path, "rb") as f:
            vectorstore = pickle.load(f)
            chain = RetrievalQAWithSourcesChain.from_llm( llm=llm, retriever= vectorstore.as_retriever())
            result = chain({"question": query}, return_only_outputs=True)

            # display answer
            st.header("Answer")
            st.write(result["answer"])

            # display source of answer
            sources = result.get("sources", "")
            if sources:
                st.subheader("Sources: ")
                sources_list = sources.split("\n") # split sources by new line
                for source in sources_list:
                    st.write(source)
langchain.text_splitter.RecursiveCharacterTextSplitter langchain.document_loaders.UnstructuredURLLoader langchain.vectorstores.FAISS.from_documents langchain.embeddings.OpenAIEmbeddings langchain.OpenAI
[((487, 518), 'streamlit.title', 'st.title', (['"""News Research tool """'], {}), "('News Research tool ')\n", (495, 518), True, 'import streamlit as st\n'), ((519, 556), 'streamlit.sidebar.title', 'st.sidebar.title', (['"""News Article URLs"""'], {}), "('News Article URLs')\n", (535, 556), True, 'import streamlit as st\n'), ((678, 711), 'streamlit.sidebar.button', 'st.sidebar.button', (['"""Process URLs"""'], {}), "('Process URLs')\n", (695, 711), True, 'import streamlit as st\n'), ((769, 779), 'streamlit.empty', 'st.empty', ([], {}), '()\n', (777, 779), True, 'import streamlit as st\n'), ((786, 825), 'langchain.OpenAI', 'OpenAI', ([], {'temperature': '(0.8)', 'max_tokens': '(500)'}), '(temperature=0.8, max_tokens=500)\n', (792, 825), False, 'from langchain import OpenAI\n'), ((596, 635), 'streamlit.sidebar.text_input', 'st.sidebar.text_input', (['f""" URL {i + 1} """'], {}), "(f' URL {i + 1} ')\n", (617, 635), True, 'import streamlit as st\n'), ((882, 914), 'langchain.document_loaders.UnstructuredURLLoader', 'UnstructuredURLLoader', ([], {'urls': 'urls'}), '(urls=urls)\n', (903, 914), False, 'from langchain.document_loaders import UnstructuredURLLoader\n'), ((1054, 1141), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'separators': "['\\nn', '\\n', '.', ' ']", 'chunk_size': '(1000)'}), "(separators=['\\nn', '\\n', '.', ' '],\n    chunk_size=1000)\n", (1084, 1141), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1320, 1338), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (1336, 1338), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((1364, 1402), 'langchain.vectorstores.FAISS.from_documents', 'FAISS.from_documents', (['docs', 'embeddings'], {}), '(docs, embeddings)\n', (1384, 1402), False, 'from langchain.vectorstores import FAISS\n'), ((1467, 1480), 'time.sleep', 'time.sleep', (['(2)'], {}), '(2)\n', (1477, 1480), False, 'import time\n'), ((1673, 1698), 'os.path.exists', 'os.path.exists', (['file_path'], {}), '(file_path)\n', (1687, 1698), False, 'import os\n'), ((1571, 1605), 'pickle.dump', 'pickle.dump', (['vectorstore_openai', 'f'], {}), '(vectorstore_openai, f)\n', (1582, 1605), False, 'import pickle\n'), ((1767, 1781), 'pickle.load', 'pickle.load', (['f'], {}), '(f)\n', (1778, 1781), False, 'import pickle\n'), ((2004, 2023), 'streamlit.header', 'st.header', (['"""Answer"""'], {}), "('Answer')\n", (2013, 2023), True, 'import streamlit as st\n'), ((2036, 2062), 'streamlit.write', 'st.write', (["result['answer']"], {}), "(result['answer'])\n", (2044, 2062), True, 'import streamlit as st\n'), ((2191, 2216), 'streamlit.subheader', 'st.subheader', (['"""Sources: """'], {}), "('Sources: ')\n", (2203, 2216), True, 'import streamlit as st\n'), ((2360, 2376), 'streamlit.write', 'st.write', (['source'], {}), '(source)\n', (2368, 2376), True, 'import streamlit as st\n')]