File size: 2,166 Bytes
7ff12f8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
import langchain
import streamlit as st
import pickle as pkl
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.document_loaders import UnstructuredURLLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import SentenceTransformerEmbeddings
from langchain.vectorstores import Chroma, FAISS
from langchain_openai import ChatOpenAI
from dotenv import load_dotenv
import time
load_dotenv("ping.env")
api_key=os.getenv("OPENAI_API_KEY")
api_base=os.getenv("OPENAI_API_BASE")
llm=ChatOpenAI(model_name="google/gemma-3n-e2b-it:free",temperature=0)
path="embedmo.pkl"
m1=pkl.load(open(path,"rb"))
st.title("URL ANALYSER🔗")
st.sidebar.title("Give your URls🔗?")
mp=st.empty()
urs = [st.sidebar.text_input(f"URL {i+1}🔗") for i in range(3)]
urs = [url for url in urs if url.strip()] # ✅ filters out empty inputs
purs=st.button("gotcha", disabled=not any(url.strip() for url in urs))
if purs:
mp.text("Loading..URl..Loader....☑️☑️☑️")
sic=UnstructuredURLLoader(urls=urs)
docs=sic.load()
mp.text("Loading..txt..splitter....☑️☑️☑️")
tot=RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=512,chunk_overlap=16)
doccs=tot.split_documents(docs)
mp.text("Loading..VB...☑️☑️☑️")
vv=FAISS.from_documents(doccs,m1)
r2=vv.as_retriever(search_type="similarity",search_kwargs={"k":4})
mp.text("Loading..Retri....☑️☑️☑️")
ra1=RetrievalQAWithSourcesChain.from_chain_type(llm=llm,retriever=r2,chain_type="map_reduce")
st.session_state.ra1=ra1
mp.text("DB & Retri Done ✅✅✅")
time.sleep(3)
query=mp.text_input("UR Question??")
if query:
if "ra1" not in st.session_state:
st.warning("pls give ur urls")
else:
with st.spinner("Wait for it..."):
result=st.session_state.ra1({"question":query},return_only_outputs=True)
st.header("Answer")
st.subheader(result["answer"])
g = st.button("Source")
if g:
sources = result.get("sources", "")
st.subheader("Sources")
for line in sources.split("\n"):
st.write(line)
|