or load from a .env file# import dotenv# dotenv.load_dotenv()# Load documentsfrom langchain.document_loaders import WebBaseLoaderloader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")# Split documentsfrom langchain.text_splitter import RecursiveCharacterTextSplittertext_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)splits = text_splitter.split_documents(loader.load())# Embed and store splitsfrom langchain.vectorstores import Chromafrom langchain.embeddings import OpenAIEmbeddingsvectorstore = Chroma.from_documents(documents=splits,embedding=OpenAIEmbeddings())retriever = vectorstore.as_retriever()# Prompt # https://smith.langchain.com/hub/rlm/rag-promptfrom langchain import hubrag_prompt = hub.pull("rlm/rag-prompt")# LLMfrom langchain.chat_models import ChatOpenAIllm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)# RAG chain from langchain.schema.runnable import RunnablePassthroughrag_chain = (    {"context": retriever, "question": RunnablePassthrough()}     | rag_prompt     | llm )rag_chain.invoke("What is Task Decomposition?")    AIMessage(content='Task decomposition is the process of breaking down a task into smaller subgoals or steps. It can be done using simple prompting, task-specific instructions, or human inputs.')Here is the LangSmith trace for this chain.Below we will explain each step in more detail.Step 1. Load‚ÄãSpecify a DocumentLoader to load in your unstructured data as Documents. A Document is a dict with text (page_content) and metadata.from langchain.document_loaders import WebBaseLoaderloader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")data = loader.load()Go deeper‚ÄãBrowse the > 160 data loader integrations here.See further documentation on loaders here.Step 2. Split‚ÄãSplit the Document into chunks for embedding and vector storage.from langchain.text_splitter import RecursiveCharacterTextSplittertext_splitter = RecursiveCharacterTextSplitter(chunk_size =
Open In Colab
Open In Colab ->: or load from a .env file# import dotenv# dotenv.load_dotenv()# Load documentsfrom langchain.document_loaders import WebBaseLoaderloader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")# Split documentsfrom langchain.text_splitter import RecursiveCharacterTextSplittertext_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)splits = text_splitter.split_documents(loader.load())# Embed and store splitsfrom langchain.vectorstores import Chromafrom langchain.embeddings import OpenAIEmbeddingsvectorstore = Chroma.from_documents(documents=splits,embedding=OpenAIEmbeddings())retriever = vectorstore.as_retriever()# Prompt # https://smith.langchain.com/hub/rlm/rag-promptfrom langchain import hubrag_prompt = hub.pull("rlm/rag-prompt")# LLMfrom langchain.chat_models import ChatOpenAIllm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)# RAG chain from langchain.schema.runnable import RunnablePassthroughrag_chain = (    {"context": retriever, "question": RunnablePassthrough()}     | rag_prompt     | llm )rag_chain.invoke("What is Task Decomposition?")    AIMessage(content='Task decomposition is the process of breaking down a task into smaller subgoals or steps. It can be done using simple prompting, task-specific instructions, or human inputs.')Here is the LangSmith trace for this chain.Below we will explain each step in more detail.Step 1. Load‚ÄãSpecify a DocumentLoader to load in your unstructured data as Documents. A Document is a dict with text (page_content) and metadata.from langchain.document_loaders import WebBaseLoaderloader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")data = loader.load()Go deeper‚ÄãBrowse the > 160 data loader integrations here.See further documentation on loaders here.Step 2. Split‚ÄãSplit the Document into chunks for embedding and vector storage.from langchain.text_splitter import RecursiveCharacterTextSplittertext_splitter = RecursiveCharacterTextSplitter(chunk_size =