loader.Attention: The loader must set source keys correctly.from langchain.document_loaders.base import BaseLoaderclass MyCustomLoader(BaseLoader):    def lazy_load(self):        text_splitter = CharacterTextSplitter(            separator="t", keep_separator=True, chunk_size=12, chunk_overlap=2        )        docs = [            Document(page_content="woof woof", metadata={"source": "doggy.txt"}),            Document(page_content="woof woof woof", metadata={"source": "doggy.txt"}),        ]        yield from text_splitter.split_documents(docs)    def load(self):        return list(self.lazy_load())_clear()loader = MyCustomLoader()loader.load()    [Document(page_content='woof woof', metadata={'source': 'doggy.txt'}),     Document(page_content='woof woof woof', metadata={'source': 'doggy.txt'})]index(loader, record_manager, vectorstore, cleanup="full", source_id_key="source")    {'num_added': 2, 'num_updated': 0, 'num_skipped': 0, 'num_deleted': 0}vectorstore.similarity_search("dog", k=30)    [Document(page_content='woof woof', metadata={'source': 'doggy.txt'}),     Document(page_content='woof woof woof', metadata={'source': 'doggy.txt'})]PreviousWebResearchRetrieverNextChainsHow it worksDeletion modesRequirementsCautionQuickstartNone deletion mode"incremental" deletion mode"full" deletion modeSourceUsing with loadersCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright ¬© 2023 LangChain, Inc.
Here, we will look at a basic indexing workflow using the LangChain indexing API.
Here, we will look at a basic indexing workflow using the LangChain indexing API. ->: loader.Attention: The loader must set source keys correctly.from langchain.document_loaders.base import BaseLoaderclass MyCustomLoader(BaseLoader):    def lazy_load(self):        text_splitter = CharacterTextSplitter(            separator="t", keep_separator=True, chunk_size=12, chunk_overlap=2        )        docs = [            Document(page_content="woof woof", metadata={"source": "doggy.txt"}),            Document(page_content="woof woof woof", metadata={"source": "doggy.txt"}),        ]        yield from text_splitter.split_documents(docs)    def load(self):        return list(self.lazy_load())_clear()loader = MyCustomLoader()loader.load()    [Document(page_content='woof woof', metadata={'source': 'doggy.txt'}),     Document(page_content='woof woof woof', metadata={'source': 'doggy.txt'})]index(loader, record_manager, vectorstore, cleanup="full", source_id_key="source")    {'num_added': 2, 'num_updated': 0, 'num_skipped': 0, 'num_deleted': 0}vectorstore.similarity_search("dog", k=30)    [Document(page_content='woof woof', metadata={'source': 'doggy.txt'}),     Document(page_content='woof woof woof', metadata={'source': 'doggy.txt'})]PreviousWebResearchRetrieverNextChainsHow it worksDeletion modesRequirementsCautionQuickstartNone deletion mode"incremental" deletion mode"full" deletion modeSourceUsing with loadersCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright ¬© 2023 LangChain, Inc.