fine-tuned metrics WIP

.gitattributes

@@ -67,3 +67,4 @@ results/mac-results_few_shots_openai.csv filter=lfs diff=lfs merge=lfs -text
 results/mac-results_fine_tuned.csv filter=lfs diff=lfs merge=lfs -text
 results/mac-results_greedy_decoding_metrics.csv filter=lfs diff=lfs merge=lfs -text
 results/mac-results_few_shots_metrics.csv filter=lfs diff=lfs merge=lfs -text
+results/mac-results_fine_tuned_metrics.csv filter=lfs diff=lfs merge=lfs -text

llm_toolkit/translation_utils.py

@@ -290,6 +290,9 @@ def get_metrics(df, max_output_tokens=2048, variant="rpp"):
 
     metrics_df["num_max_output_tokens"] = num_max_output_tokens
 
+    if variant != "rpp":
+        metrics_df[variant] = metrics_df[variant].astype(int)
+
     return metrics_df
 
 
@@ -510,11 +513,10 @@ def convert_time_to_seconds(time_str):
     return total_seconds
 
 
-time_pattern = re.compile(r"\[(.{5,10})<00:00")
-metrics_pattern = re.compile(r"(.*)/shots-(.*) metrics:")
-
+def process_log_file(log_file, total_entries, variant):
+    time_pattern = re.compile(r"\[(.{5,10})<00:00")
+    metrics_pattern = re.compile(rf"(.*)/{variant}-(.*) metrics:")
 
-def process_log_file(log_file, total_entries):
     model = []
     shots = []
     eval_time = []
@@ -546,27 +548,28 @@ def process_log_file(log_file, total_entries):
     df = pd.DataFrame(
         {
             "model": model,
-            "shots": shots,
+            variant: shots,
             "eval_time": eval_time,
         }
     )
     return df
 
 
-def load_eval_times(logs_folder, total_entries=1133):
+def load_eval_times(logs_folder, total_entries=1133, variant="shots"):
     # Get a list of all files in the logs folder
     log_files = glob.glob(os.path.join(logs_folder, "*"))
     log_files.sort()
 
-    time_df = pd.DataFrame({"model": [], "shots": [], "eval_time": []})
+    time_df = pd.DataFrame({"model": [], variant: [], "eval_time": []})
 
     for log_file in log_files:
         print(f"Loading content of {log_file}")
-        df = process_log_file(log_file, total_entries=total_entries)
+        df = process_log_file(log_file, total_entries, variant)
         time_df = pd.concat([time_df, df], ignore_index=True)
 
-    time_df["shots"] = time_df["shots"].apply(lambda x: int(x))
-    return time_df
+    time_df[variant] = time_df[variant].apply(lambda x: int(x))
+    # Keep the last occurrence of each duplicate
+    return time_df.drop_duplicates(subset=["model", variant], keep="last")
 
 
 def load_alpaca_data(data_path):

notebooks/00b_Data Analysis_Few_Shots.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab12535095ccc1615c5379d154c81142686e4fa3dc33fc51808b679fd6fa7116
-size 1234635
+oid sha256:79fc6f45be27f13fe14be2e41598b8cc605cfcb71565a136603608711a5338fa
+size 1606476

results/mac-results_few_shots_metrics.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da524c8a2bcc6e996545af1c48cf428675203bbf299fea395f8679418b599013
-size 7931
+oid sha256:40bd68a7831cf37a0bd5b4e290435873c574a15bca5a79400127a46ff2717672
+size 8156

results/mac-results_fine_tuned_metrics.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:506cb6b5562efe12e50f6cd3c20ee50c979d5c6ef344ce933ca52f8ef26159fa
+size 3217