Add LLM Developer source to project

data/scraping_scripts/add_context_to_nodes.py

@@ -162,13 +162,6 @@ async def process(
 
     results: List[TextNode] = await tqdm.gather(*tasks, desc="Processing chunks")
 
-    # results: List[TextNode] = []
-    # # Add tqdm progress bar with semaphore limit
-    # for task in tqdm(
-    #     asyncio.as_completed(tasks), total=len(tasks), desc="Processing chunks"
-    # ):
-    #     result = await task
-    #     results.append(result)
     # pdb.set_trace()
 
     return results

data/scraping_scripts/process_md_files.py

@@ -416,6 +416,18 @@ SOURCE_CONFIGS = {
         "included_root_files": [],
         "url_extension": "",
     },
+    "llm_developer": {
+        "base_url": "",
+        "input_directory": "data/llm_developer",
+        "output_file": "data/llm_developer_data.jsonl",  # From Beginner to Advanced LLM Developer
+        "source_name": "llm_developer",
+        "use_include_list": False,
+        "included_dirs": [],
+        "excluded_dirs": [],
+        "excluded_root_files": [],
+        "included_root_files": [],
+        "url_extension": "",
+    },
 }
 
 

scripts/main.py

@@ -129,6 +129,7 @@ def generate_completion(
             "OpenAI Cookbooks": "openai_cookbooks",
             "Towards AI Blog": "tai_blog",
             "8 Hour Primer": "8-hour_primer",
+            "Advanced LLM Developer": "llm_developer",
         }
 
         for source in sources:
@@ -245,6 +246,7 @@ sources = gr.CheckboxGroup(
         "OpenAI Cookbooks",
         "Towards AI Blog",
         "8 Hour Primer",
+        "Advanced LLM Developer",
         # "All Sources",
     ],
     interactive=True,

scripts/setup.py

@@ -178,6 +178,7 @@ AVAILABLE_SOURCES_UI = [
     "OpenAI Cookbooks",
     "Towards AI Blog",
     "8 Hour Primer",
+    "Advanced LLM Developer",
     # "All Sources",
 ]
 
@@ -190,6 +191,7 @@ AVAILABLE_SOURCES = [
     "openai_cookbooks",
     "tai_blog",
     "8-hour_primer",
+    "llm_developer",
     # "all_sources",
 ]