Add an experiment tag so the evaluation can be done on a specific run only. evaluate_on_dataset.py auto-generates the tag.

experiments/evaluate_on_dataset.py

@@ -8,6 +8,7 @@ from preprocess.html_to_documents import extract_documents_from_html
 from langchain_openai import ChatOpenAI
 from pathlib import Path
 import pickle
+import uuid
 
 
 load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'))
@@ -15,6 +16,8 @@ load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'))
 # --- CONFIG ---
 DATASET_NAME = "State of AI Across the Years!"
 PROJECT_NAME = "State of AI Across the Years!"
+EXPERIMENT_TAG = f"exp_{uuid.uuid4().hex[:8]}"
+print(f"Experiment tag for this batch: {EXPERIMENT_TAG}")
 
 # --- LOAD DOCUMENTS & VECTORSTORE ---
 def load_docs():
@@ -62,7 +65,9 @@ def main():
             reference_outputs={"answer": reference},
             example_id=example.id,
             project_name=PROJECT_NAME,
+            metadata={"experiment_tag": EXPERIMENT_TAG},
         )
+        print(f"Logged run with experiment_tag: {EXPERIMENT_TAG}")
         print(f"Processed: {question}\n  → {result.answer}\n")
 
 if __name__ == "__main__":

experiments/evaluate_predictions.py

@@ -2,6 +2,8 @@ import os
 from dotenv import load_dotenv
 from langsmith.evaluation import LangChainStringEvaluator, evaluate
 from langchain_openai import ChatOpenAI
+import argparse
+from langsmith import Client
 
 
 load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), '..', '.env'))
@@ -11,6 +13,20 @@ DATASET_NAME = "State of AI Across the Years!"
 PROJECT_NAME = "State of AI Across the Years!"
 EVAL_LLM_MODEL = "gpt-4.1"  # Match the notebook's model if possible
 
+parser = argparse.ArgumentParser()
+parser.add_argument("--experiment_tag", type=str, help="Only evaluate runs with this experiment_tag")
+args = parser.parse_args()
+
+if args.experiment_tag:
+    print(f"Evaluating only runs with experiment_tag: {args.experiment_tag}")
+
+client = Client()
+runs = list(client.list_runs(
+    project_name=PROJECT_NAME,
+    dataset_name=DATASET_NAME,
+    filters={"metadata.experiment_tag": args.experiment_tag} if args.experiment_tag else None,
+))
+
 # --- EVALUATORS ---
 eval_llm = ChatOpenAI(model=EVAL_LLM_MODEL)
 
@@ -48,7 +64,7 @@ dope_or_nope_evaluator = LangChainStringEvaluator(
 if __name__ == "__main__":
     print("Running evaluation on predictions in LangSmith...")
     results = evaluate(
-        None,  # No need to pass a chain, just evaluate existing runs
+        runs,
         data=DATASET_NAME,
         evaluators=[
             qa_evaluator,