transformersText embedding modelsVector storesRetrieversToolsAgents and toolkitsMemoryCallbacksChat loadersProvidersMoreVectaraVectara Text GenerationOn this pageVectara Text GenerationThis notebook is based on text generation notebook and adapted to Vectara.Prepare Data‚ÄãFirst, we prepare the data. For this example, we fetch a documentation site that consists of markdown files hosted on Github and split them into small enough Documents.import osfrom langchain.llms import OpenAIfrom langchain.docstore.document import Documentimport requestsfrom langchain.vectorstores import Vectarafrom langchain.text_splitter import CharacterTextSplitterfrom langchain.prompts import PromptTemplateimport pathlibimport subprocessimport tempfiledef get_github_docs(repo_owner, repo_name):    with tempfile.TemporaryDirectory() as d:        subprocess.check_call(            f"git clone --depth 1 https://github.com/{repo_owner}/{repo_name}.git .",            cwd=d,            shell=True,        )        git_sha = (            subprocess.check_output("git rev-parse HEAD", shell=True, cwd=d)            .decode("utf-8")            .strip()        )        repo_path = pathlib.Path(d)        markdown_files = list(repo_path.glob("*/*.md")) + list(            repo_path.glob("*/*.mdx")        )        for markdown_file in markdown_files:            with open(markdown_file, "r") as f:                relative_path = markdown_file.relative_to(repo_path)                github_url = f"https://github.com/{repo_owner}/{repo_name}/blob/{git_sha}/{relative_path}"                yield Document(page_content=f.read(), metadata={"source": github_url})sources = get_github_docs("yirenlu92", "deno-manual-forked")source_chunks = []splitter = CharacterTextSplitter(separator=" ", chunk_size=1024, chunk_overlap=0)for source in sources:    for chunk in splitter.split_text(source.page_content):        source_chunks.append(chunk)    Cloning into '.'...Set Up Vector DB‚ÄãNow that we have the documentation content in
This notebook is based on text generation notebook and adapted to Vectara.
This notebook is based on text generation notebook and adapted to Vectara. ->: transformersText embedding modelsVector storesRetrieversToolsAgents and toolkitsMemoryCallbacksChat loadersProvidersMoreVectaraVectara Text GenerationOn this pageVectara Text GenerationThis notebook is based on text generation notebook and adapted to Vectara.Prepare Data‚ÄãFirst, we prepare the data. For this example, we fetch a documentation site that consists of markdown files hosted on Github and split them into small enough Documents.import osfrom langchain.llms import OpenAIfrom langchain.docstore.document import Documentimport requestsfrom langchain.vectorstores import Vectarafrom langchain.text_splitter import CharacterTextSplitterfrom langchain.prompts import PromptTemplateimport pathlibimport subprocessimport tempfiledef get_github_docs(repo_owner, repo_name):    with tempfile.TemporaryDirectory() as d:        subprocess.check_call(            f"git clone --depth 1 https://github.com/{repo_owner}/{repo_name}.git .",            cwd=d,            shell=True,        )        git_sha = (            subprocess.check_output("git rev-parse HEAD", shell=True, cwd=d)            .decode("utf-8")            .strip()        )        repo_path = pathlib.Path(d)        markdown_files = list(repo_path.glob("*/*.md")) + list(            repo_path.glob("*/*.mdx")        )        for markdown_file in markdown_files:            with open(markdown_file, "r") as f:                relative_path = markdown_file.relative_to(repo_path)                github_url = f"https://github.com/{repo_owner}/{repo_name}/blob/{git_sha}/{relative_path}"                yield Document(page_content=f.read(), metadata={"source": github_url})sources = get_github_docs("yirenlu92", "deno-manual-forked")source_chunks = []splitter = CharacterTextSplitter(separator=" ", chunk_size=1024, chunk_overlap=0)for source in sources:    for chunk in splitter.split_text(source.page_content):        source_chunks.append(chunk)    Cloning into '.'...Set Up Vector DB‚ÄãNow that we have the documentation content in