import os
import streamlit as st
import time
import langchain
from langchain.chains import RetrievalQAWithSourcesChain, RetrievalQA
from langchain.chains.qa_with_sources.loading import load_qa_with_sources_chain
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_community.vectorstores import FAISS
from langchain_openai import OpenAI, OpenAIEmbeddings
import configparser

config = configparser.RawConfigParser()
config.read('../config.config')
openapi_key = config.get('Keys', 'openapi_key')
os.environ['OPENAI_API_KEY'] = openapi_key

st.title("URL Insighter 🔗🔍")
st.sidebar.title("🔗URLs...")

urls = []
for i in range(3):
    url = st.sidebar.text_input(f"URL {i+1}")
    urls.append(url)

folder_name = st.sidebar.text_input('Title')
process_url_clicked = st.sidebar.button("Process URLs")

main_placeholder = st.empty()
llm = OpenAI(temperature=0.9, max_tokens=500)

if process_url_clicked:
    # load data
    loader = UnstructuredURLLoader(urls=urls)
    main_placeholder.text("Data Loading...Started...✅✅✅")
    data = loader.load()
    # split data
    text_splitter = RecursiveCharacterTextSplitter(
        separators=['\n\n', '\n', '.', ','],
        chunk_size=1000
    )
    main_placeholder.text("Text Splitter...Started...✅✅✅")
    docs = text_splitter.split_documents(data)
    embeddings = OpenAIEmbeddings()
    vectorstore_openai = FAISS.from_documents(docs, embeddings)
    main_placeholder.text("Embedding Vector Started Building...✅✅✅")
    time.sleep(2)

    vectorstore_openai.save_local(folder_name)

query = main_placeholder.text_input("Question: ")
if query:
    if os.path.exists(folder_name):
        vectorstore = FAISS.load_local(folder_name, OpenAIEmbeddings())
        chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=vectorstore.as_retriever())
        result = chain({"question": query}, return_only_outputs=True)
        # result will be a dictionary of this format --> {"answer": "", "sources": [] }
        st.header("Answer")
        st.write(result["answer"])

        sources = result.get("sources", "")
        if sources:
            st.subheader("Sources:")
            sources_list = sources.split("\n")  # Split the sources by newline
            for source in sources_list:
                st.write(source)

langchain.text_splitter.RecursiveCharacterTextSplitter langchain_openai.OpenAIEmbeddings langchain_community.document_loaders.UnstructuredURLLoader langchain_community.vectorstores.FAISS.from_documents langchain_openai.OpenAI
[((485, 515), 'configparser.RawConfigParser', 'configparser.RawConfigParser', ([], {}), '()\n', (513, 515), False, 'import configparser\n'), ((640, 668), 'streamlit.title', 'st.title', (['"""URL Insighter 🔗🔍"""'], {}), "('URL Insighter 🔗🔍')\n", (648, 668), True, 'import streamlit as st\n'), ((669, 697), 'streamlit.sidebar.title', 'st.sidebar.title', (['"""🔗URLs..."""'], {}), "('🔗URLs...')\n", (685, 697), True, 'import streamlit as st\n'), ((810, 840), 'streamlit.sidebar.text_input', 'st.sidebar.text_input', (['"""Title"""'], {}), "('Title')\n", (831, 840), True, 'import streamlit as st\n'), ((863, 896), 'streamlit.sidebar.button', 'st.sidebar.button', (['"""Process URLs"""'], {}), "('Process URLs')\n", (880, 896), True, 'import streamlit as st\n'), ((917, 927), 'streamlit.empty', 'st.empty', ([], {}), '()\n', (925, 927), True, 'import streamlit as st\n'), ((934, 973), 'langchain_openai.OpenAI', 'OpenAI', ([], {'temperature': '(0.9)', 'max_tokens': '(500)'}), '(temperature=0.9, max_tokens=500)\n', (940, 973), False, 'from langchain_openai import OpenAI, OpenAIEmbeddings\n'), ((738, 775), 'streamlit.sidebar.text_input', 'st.sidebar.text_input', (['f"""URL {i + 1}"""'], {}), "(f'URL {i + 1}')\n", (759, 775), True, 'import streamlit as st\n'), ((1028, 1060), 'langchain_community.document_loaders.UnstructuredURLLoader', 'UnstructuredURLLoader', ([], {'urls': 'urls'}), '(urls=urls)\n', (1049, 1060), False, 'from langchain_community.document_loaders import UnstructuredURLLoader\n'), ((1181, 1269), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'separators': "['\\n\\n', '\\n', '.', ',']", 'chunk_size': '(1000)'}), "(separators=['\\n\\n', '\\n', '.', ','],\n    chunk_size=1000)\n", (1211, 1269), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1411, 1429), 'langchain_openai.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (1427, 1429), False, 'from langchain_openai import OpenAI, OpenAIEmbeddings\n'), ((1455, 1493), 'langchain_community.vectorstores.FAISS.from_documents', 'FAISS.from_documents', (['docs', 'embeddings'], {}), '(docs, embeddings)\n', (1475, 1493), False, 'from langchain_community.vectorstores import FAISS\n'), ((1567, 1580), 'time.sleep', 'time.sleep', (['(2)'], {}), '(2)\n', (1577, 1580), False, 'import time\n'), ((1697, 1724), 'os.path.exists', 'os.path.exists', (['folder_name'], {}), '(folder_name)\n', (1711, 1724), False, 'import os\n'), ((2064, 2083), 'streamlit.header', 'st.header', (['"""Answer"""'], {}), "('Answer')\n", (2073, 2083), True, 'import streamlit as st\n'), ((2092, 2118), 'streamlit.write', 'st.write', (["result['answer']"], {}), "(result['answer'])\n", (2100, 2118), True, 'import streamlit as st\n'), ((1778, 1796), 'langchain_openai.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (1794, 1796), False, 'from langchain_openai import OpenAI, OpenAIEmbeddings\n'), ((2196, 2220), 'streamlit.subheader', 'st.subheader', (['"""Sources:"""'], {}), "('Sources:')\n", (2208, 2220), True, 'import streamlit as st\n'), ((2356, 2372), 'streamlit.write', 'st.write', (['source'], {}), '(source)\n', (2364, 2372), True, 'import streamlit as st\n')]