# Ingest Documents into a Zep Collection
import os

from dotenv import find_dotenv, load_dotenv
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import WebBaseLoader

from zep_python import ZepClient
from zep_python.langchain.vectorstore import ZepVectorStore

load_dotenv(dotenv_path=find_dotenv())

SOURCE = "https://en.wikipedia.org/wiki/Leonard_Bernstein"  # noqa: E501

ZEP_API_URL = os.environ.get(
    "ZEP_API_URL"
)  # only required if you're using Zep Open Source

ZEP_API_KEY = os.environ.get("ZEP_API_KEY")  # Required for Zep Cloud
if ZEP_API_KEY is None:
    raise ValueError(
        "ZEP_API_KEY is required for Zep Cloud. "
        "Remove this check if using Zep Open Source."
    )

ZEP_COLLECTION_NAME = os.environ.get("ZEP_COLLECTION_NAME")
if ZEP_COLLECTION_NAME is None:
    raise ValueError("ZEP_COLLECTION_NAME is required for ingestion. ")

zep = ZepClient(
    api_key=ZEP_API_KEY,
    api_url=ZEP_API_URL,  # only required if you're using Zep Open Source
)

# Load
loader = WebBaseLoader(SOURCE)
data = loader.load()

print(f"Loaded: {len(data)} documents")

# Split
text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=200)
all_splits = text_splitter.split_documents(data)

print(f"Adding {len(all_splits)} documents to {ZEP_COLLECTION_NAME}...")

# Add to vectorDB
vectorstore = ZepVectorStore.from_documents(
    documents=all_splits,
    collection_name=ZEP_COLLECTION_NAME,
    zep_client=zep,
)

print(f"Added {len(all_splits)} documents to {ZEP_COLLECTION_NAME}...")

langchain_community.document_loaders.WebBaseLoader langchain.text_splitter.RecursiveCharacterTextSplitter
[((449, 478), 'os.environ.get', 'os.environ.get', (['"""ZEP_API_URL"""'], {}), "('ZEP_API_URL')\n", (463, 478), False, 'import os\n'), ((549, 578), 'os.environ.get', 'os.environ.get', (['"""ZEP_API_KEY"""'], {}), "('ZEP_API_KEY')\n", (563, 578), False, 'import os\n'), ((784, 821), 'os.environ.get', 'os.environ.get', (['"""ZEP_COLLECTION_NAME"""'], {}), "('ZEP_COLLECTION_NAME')\n", (798, 821), False, 'import os\n'), ((933, 984), 'zep_python.ZepClient', 'ZepClient', ([], {'api_key': 'ZEP_API_KEY', 'api_url': 'ZEP_API_URL'}), '(api_key=ZEP_API_KEY, api_url=ZEP_API_URL)\n', (942, 984), False, 'from zep_python import ZepClient\n'), ((1062, 1083), 'langchain_community.document_loaders.WebBaseLoader', 'WebBaseLoader', (['SOURCE'], {}), '(SOURCE)\n', (1075, 1083), False, 'from langchain_community.document_loaders import WebBaseLoader\n'), ((1171, 1236), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(400)', 'chunk_overlap': '(200)'}), '(chunk_size=400, chunk_overlap=200)\n', (1201, 1236), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1393, 1502), 'zep_python.langchain.vectorstore.ZepVectorStore.from_documents', 'ZepVectorStore.from_documents', ([], {'documents': 'all_splits', 'collection_name': 'ZEP_COLLECTION_NAME', 'zep_client': 'zep'}), '(documents=all_splits, collection_name=\n    ZEP_COLLECTION_NAME, zep_client=zep)\n', (1422, 1502), False, 'from zep_python.langchain.vectorstore import ZepVectorStore\n'), ((345, 358), 'dotenv.find_dotenv', 'find_dotenv', ([], {}), '()\n', (356, 358), False, 'from dotenv import find_dotenv, load_dotenv\n')]