covers how to load source code files using a special approach with language parsing: each top-level function and class in the code is loaded into separate documents. Any remaining code top-level code outside the already loaded functions and classes will be loaded into a separate document.This approach can potentially improve the accuracy of QA models over source code. Currently, the supported languages for code parsing are Python and JavaScript. The language used for parsing can be configured, along with the minimum number of lines required to activate the splitting based on syntax.pip install esprimaimport warningswarnings.filterwarnings("ignore")from pprint import pprintfrom langchain.text_splitter import Languagefrom langchain.document_loaders.generic import GenericLoaderfrom langchain.document_loaders.parsers import LanguageParserloader = GenericLoader.from_filesystem(    "./example_data/source_code",    glob="*",    suffixes=[".py", ".js"],    parser=LanguageParser(),)docs = loader.load()len(docs)    6for document in docs:    pprint(document.metadata)    {'content_type': 'functions_classes',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'functions_classes',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'simplified_code',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'functions_classes',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}    {'content_type': 'functions_classes',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}    {'content_type': 'simplified_code',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}print("\n\n--8<--\n\n".join([document.page_content for document in docs]))    class MyClass:        def __init__(self,
This notebook covers how to load source code files using a special approach with language parsing: each top-level function and class in the code is loaded into separate documents. Any remaining code top-level code outside the already loaded functions and classes will be loaded into a separate document.
This notebook covers how to load source code files using a special approach with language parsing: each top-level function and class in the code is loaded into separate documents. Any remaining code top-level code outside the already loaded functions and classes will be loaded into a separate document. ->: covers how to load source code files using a special approach with language parsing: each top-level function and class in the code is loaded into separate documents. Any remaining code top-level code outside the already loaded functions and classes will be loaded into a separate document.This approach can potentially improve the accuracy of QA models over source code. Currently, the supported languages for code parsing are Python and JavaScript. The language used for parsing can be configured, along with the minimum number of lines required to activate the splitting based on syntax.pip install esprimaimport warningswarnings.filterwarnings("ignore")from pprint import pprintfrom langchain.text_splitter import Languagefrom langchain.document_loaders.generic import GenericLoaderfrom langchain.document_loaders.parsers import LanguageParserloader = GenericLoader.from_filesystem(    "./example_data/source_code",    glob="*",    suffixes=[".py", ".js"],    parser=LanguageParser(),)docs = loader.load()len(docs)    6for document in docs:    pprint(document.metadata)    {'content_type': 'functions_classes',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'functions_classes',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'simplified_code',     'language': <Language.PYTHON: 'python'>,     'source': 'example_data/source_code/example.py'}    {'content_type': 'functions_classes',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}    {'content_type': 'functions_classes',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}    {'content_type': 'simplified_code',     'language': <Language.JS: 'js'>,     'source': 'example_data/source_code/example.js'}print("\n\n--8<--\n\n".join([document.page_content for document in docs]))    class MyClass:        def __init__(self,