|
|
import langchain |
|
|
import streamlit as st |
|
|
import pickle as pkl |
|
|
from langchain.chains import RetrievalQAWithSourcesChain |
|
|
from langchain.document_loaders import UnstructuredURLLoader |
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter |
|
|
from langchain.embeddings import SentenceTransformerEmbeddings |
|
|
from langchain.vectorstores import Chroma, FAISS |
|
|
from langchain_openai import ChatOpenAI |
|
|
from dotenv import load_dotenv |
|
|
import time |
|
|
|
|
|
load_dotenv("ping.env") |
|
|
api_key=os.getenv("OPENAI_API_KEY") |
|
|
api_base=os.getenv("OPENAI_API_BASE") |
|
|
|
|
|
llm=ChatOpenAI(model_name="google/gemma-3n-e2b-it:free",temperature=0) |
|
|
|
|
|
path="embedmo.pkl" |
|
|
m1=pkl.load(open(path,"rb")) |
|
|
|
|
|
st.title("URL ANALYSER🔗") |
|
|
st.sidebar.title("Give your URls🔗?") |
|
|
mp=st.empty() |
|
|
|
|
|
|
|
|
urs = [st.sidebar.text_input(f"URL {i+1}🔗") for i in range(3)] |
|
|
urs = [url for url in urs if url.strip()] |
|
|
|
|
|
purs=st.button("gotcha", disabled=not any(url.strip() for url in urs)) |
|
|
if purs: |
|
|
mp.text("Loading..URl..Loader....☑️☑️☑️") |
|
|
sic=UnstructuredURLLoader(urls=urs) |
|
|
docs=sic.load() |
|
|
mp.text("Loading..txt..splitter....☑️☑️☑️") |
|
|
tot=RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=512,chunk_overlap=16) |
|
|
doccs=tot.split_documents(docs) |
|
|
mp.text("Loading..VB...☑️☑️☑️") |
|
|
vv=FAISS.from_documents(doccs,m1) |
|
|
r2=vv.as_retriever(search_type="similarity",search_kwargs={"k":4}) |
|
|
mp.text("Loading..Retri....☑️☑️☑️") |
|
|
ra1=RetrievalQAWithSourcesChain.from_chain_type(llm=llm,retriever=r2,chain_type="map_reduce") |
|
|
st.session_state.ra1=ra1 |
|
|
mp.text("DB & Retri Done ✅✅✅") |
|
|
time.sleep(3) |
|
|
query=mp.text_input("UR Question??") |
|
|
if query: |
|
|
if "ra1" not in st.session_state: |
|
|
st.warning("pls give ur urls") |
|
|
else: |
|
|
with st.spinner("Wait for it..."): |
|
|
result=st.session_state.ra1({"question":query},return_only_outputs=True) |
|
|
st.header("Answer") |
|
|
st.subheader(result["answer"]) |
|
|
g = st.button("Source") |
|
|
if g: |
|
|
sources = result.get("sources", "") |
|
|
st.subheader("Sources") |
|
|
for line in sources.split("\n"): |
|
|
st.write(line) |
|
|
|
|
|
|
|
|
|
