import os
import gradio as gr
import langchain
from langchain.llms import OpenAI
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.chains.qa_with_sources.loading import load_qa_with_sources_chain
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import UnstructuredURLLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from dotenv import load_dotenv


load_dotenv() # take .env variables
llm = OpenAI(temperature=0.9, max_tokens=500)

def echo(message, history, links):
    question = message
    urls = links.split()
    loader = UnstructuredURLLoader(urls)
    data = loader.load()
    
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    docs = text_splitter.split_documents(data)
    
    vectorindex_openai = FAISS.from_documents(docs, OpenAIEmbeddings())
    chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=vectorindex_openai.as_retriever())

    response = chain({"question": question}, return_only_outputs=True)
    formatted_response = response['answer']
    if 'sources' in response:
        formatted_response += "\nSources: " + response['sources']

    return formatted_response

demo = gr.ChatInterface(echo, 
                        additional_inputs=[
                            gr.Textbox("[Paste Links Here]", label="News Links"),
                        ]
                       )
    
if __name__ == "__main__":
    demo.launch(show_api=False)   
    
    
langchain.text_splitter.RecursiveCharacterTextSplitter langchain.embeddings.OpenAIEmbeddings langchain.llms.OpenAI langchain.document_loaders.UnstructuredURLLoader
[((470, 483), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (481, 483), False, 'from dotenv import load_dotenv\n'), ((512, 551), 'langchain.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0.9)', 'max_tokens': '(500)'}), '(temperature=0.9, max_tokens=500)\n', (518, 551), False, 'from langchain.llms import OpenAI\n'), ((649, 676), 'langchain.document_loaders.UnstructuredURLLoader', 'UnstructuredURLLoader', (['urls'], {}), '(urls)\n', (670, 676), False, 'from langchain.document_loaders import UnstructuredURLLoader\n'), ((727, 793), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(1000)', 'chunk_overlap': '(200)'}), '(chunk_size=1000, chunk_overlap=200)\n', (757, 793), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((898, 916), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (914, 916), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((1368, 1420), 'gradio.Textbox', 'gr.Textbox', (['"""[Paste Links Here]"""'], {'label': '"""News Links"""'}), "('[Paste Links Here]', label='News Links')\n", (1378, 1420), True, 'import gradio as gr\n')]