import os
import streamlit as st
from PyPDF2 import PdfReader
import langchain
langchain.verbose = False
from langchain.text_splitter import CharacterTextSplitter
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import OpenAI
from langchain.callbacks import get_openai_callback
import requests
from bs4 import BeautifulSoup

def webscrap(name):
    # Replace this URL with the one you want to scrape
    url = f'https://www.{name}.com'

    response = requests.get(url)

    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        page_text = soup.get_text()
        return page_text
    else:
        return None
    
def main():
    print(os.getenv('OPENAI_API_KEY'))
    
    st.set_page_config(page_title="Webscrap chatbot")
    st.header("Webscrap chatbot")

    name = st.text_input("enter website name")
    
    web_data= webscrap(name)
    if web_data is not None:
        
        text = web_data
        # for page in pdf_reader.pages:
        #     text += page.extract_text()


        max_length = 1800
        original_string = text
        temp_string = ""
        strings_list = []

        for character in original_string:
            if len(temp_string) < max_length:
                temp_string += character
            else:
                strings_list.append(temp_string)
                temp_string = ""

        if temp_string:
            strings_list.append(temp_string)

        #split into chunks
        

        # create embeddings
        embeddings = OpenAIEmbeddings()
        knowledge_base = FAISS.from_texts(strings_list, embedding=embeddings)

        user_question = st.text_input("Ask a question about your PDF")

        if user_question:
            docs = knowledge_base.similarity_search(user_question)
            
            llm = OpenAI(model_name="gpt-3.5-turbo", temperature=0.9)

            chain = load_qa_chain(llm, chain_type="stuff")
            with get_openai_callback() as cb:
                response = chain.run(input_documents = docs, question = user_question)
                print(cb)

            st.write(response)


if __name__ == '__main__':
    main()

langchain.chains.question_answering.load_qa_chain langchain.llms.OpenAI langchain.callbacks.get_openai_callback langchain.vectorstores.FAISS.from_texts langchain.embeddings.openai.OpenAIEmbeddings
[((583, 600), 'requests.get', 'requests.get', (['url'], {}), '(url)\n', (595, 600), False, 'import requests\n'), ((853, 902), 'streamlit.set_page_config', 'st.set_page_config', ([], {'page_title': '"""Webscrap chatbot"""'}), "(page_title='Webscrap chatbot')\n", (871, 902), True, 'import streamlit as st\n'), ((907, 936), 'streamlit.header', 'st.header', (['"""Webscrap chatbot"""'], {}), "('Webscrap chatbot')\n", (916, 936), True, 'import streamlit as st\n'), ((949, 984), 'streamlit.text_input', 'st.text_input', (['"""enter website name"""'], {}), "('enter website name')\n", (962, 984), True, 'import streamlit as st\n'), ((653, 696), 'bs4.BeautifulSoup', 'BeautifulSoup', (['response.text', '"""html.parser"""'], {}), "(response.text, 'html.parser')\n", (666, 696), False, 'from bs4 import BeautifulSoup\n'), ((815, 842), 'os.getenv', 'os.getenv', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (824, 842), False, 'import os\n'), ((1660, 1678), 'langchain.embeddings.openai.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (1676, 1678), False, 'from langchain.embeddings.openai import OpenAIEmbeddings\n'), ((1704, 1756), 'langchain.vectorstores.FAISS.from_texts', 'FAISS.from_texts', (['strings_list'], {'embedding': 'embeddings'}), '(strings_list, embedding=embeddings)\n', (1720, 1756), False, 'from langchain.vectorstores import FAISS\n'), ((1782, 1828), 'streamlit.text_input', 'st.text_input', (['"""Ask a question about your PDF"""'], {}), "('Ask a question about your PDF')\n", (1795, 1828), True, 'import streamlit as st\n'), ((1954, 2005), 'langchain.llms.OpenAI', 'OpenAI', ([], {'model_name': '"""gpt-3.5-turbo"""', 'temperature': '(0.9)'}), "(model_name='gpt-3.5-turbo', temperature=0.9)\n", (1960, 2005), False, 'from langchain.llms import OpenAI\n'), ((2027, 2065), 'langchain.chains.question_answering.load_qa_chain', 'load_qa_chain', (['llm'], {'chain_type': '"""stuff"""'}), "(llm, chain_type='stuff')\n", (2040, 2065), False, 'from langchain.chains.question_answering import load_qa_chain\n'), ((2238, 2256), 'streamlit.write', 'st.write', (['response'], {}), '(response)\n', (2246, 2256), True, 'import streamlit as st\n'), ((2083, 2104), 'langchain.callbacks.get_openai_callback', 'get_openai_callback', ([], {}), '()\n', (2102, 2104), False, 'from langchain.callbacks import get_openai_callback\n')]