"""Load markdown, html, text from files, clean up, split, ingest into Pinecone."""
import pinecone
import tiktoken
from langchain.document_loaders import ReadTheDocsLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import NLTKTextSplitter
from langchain.vectorstores.pinecone import Pinecone


def ingest_docs():
    """Get documents from web pages."""
    loader = ReadTheDocsLoader("hasura.io/docs/latest/")
    raw_documents = loader.load()
    text_splitter = NLTKTextSplitter.from_tiktoken_encoder(
        chunk_size=800,
        chunk_overlap=400,
    )
    documents = text_splitter.split_documents(raw_documents)
    embeddings = OpenAIEmbeddings()
    pinecone.init(
        api_key="YOUR_API_KEY",  # find at app.pinecone.io
        environment="YOUR_ENV",  # next to api key in console
    )
    Pinecone.from_documents(documents, embeddings)


if __name__ == "__main__":
    ingest_docs()

langchain.document_loaders.ReadTheDocsLoader langchain.embeddings.OpenAIEmbeddings langchain.vectorstores.pinecone.Pinecone.from_documents langchain.text_splitter.NLTKTextSplitter.from_tiktoken_encoder
[((402, 445), 'langchain.document_loaders.ReadTheDocsLoader', 'ReadTheDocsLoader', (['"""hasura.io/docs/latest/"""'], {}), "('hasura.io/docs/latest/')\n", (419, 445), False, 'from langchain.document_loaders import ReadTheDocsLoader\n'), ((500, 573), 'langchain.text_splitter.NLTKTextSplitter.from_tiktoken_encoder', 'NLTKTextSplitter.from_tiktoken_encoder', ([], {'chunk_size': '(800)', 'chunk_overlap': '(400)'}), '(chunk_size=800, chunk_overlap=400)\n', (538, 573), False, 'from langchain.text_splitter import NLTKTextSplitter\n'), ((675, 693), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (691, 693), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((698, 759), 'pinecone.init', 'pinecone.init', ([], {'api_key': '"""YOUR_API_KEY"""', 'environment': '"""YOUR_ENV"""'}), "(api_key='YOUR_API_KEY', environment='YOUR_ENV')\n", (711, 759), False, 'import pinecone\n'), ((844, 890), 'langchain.vectorstores.pinecone.Pinecone.from_documents', 'Pinecone.from_documents', (['documents', 'embeddings'], {}), '(documents, embeddings)\n', (867, 890), False, 'from langchain.vectorstores.pinecone import Pinecone\n')]