updated lb metrics

leaderboard.py

@@ -1,5 +1,5 @@
 from glob import glob
-from sklearn.metrics import accuracy_score,  recall_score
+from sklearn.metrics import accuracy_score,  recall_score, f1_score
 import os
 import pandas as pd 
 
@@ -56,7 +56,7 @@ def get_duration_scores(df):
   lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Accuracy": acc_scores})
   return lb
 
-def get_algorithm_scores(df):
+def get_algorithm_scores_v1(df):
     
     columns = list(df[df.label != 'real'].algorithm.unique())
     samples_tested = []
@@ -75,8 +75,52 @@ def get_algorithm_scores(df):
     lb = pd.DataFrame({"Sample": columns, "Num Samples": samples_tested, "Recall": rec_scores})
     return lb
 
+def get_algorithm_scores_v2(df):
+    
+    columns = list(df[df.label != 'real'].algorithm.unique())
+    columns2 = list(df[df.label != 'real'].label.unique())
+    samples_tested = []
+    acc_scores = []
+    tpr_scores = []
+    tnr_scores = [float('nan')]*(len(columns) + len(columns2))
+    f1_scores = [float('nan')]*(len(columns) + len(columns2))
+    
+    for c in columns:
+      mask = (df.algorithm == c)
+      sel_df = df[mask]
+    
+      samples_tested.append(len(sel_df))
+      tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
+    
+    
+    for c in columns2:
+      mask = (df.label == c)
+      sel_df = df[mask]
+    
+      samples_tested.append(len(sel_df))
+      tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
+    
+    mask = (df.label != "real")
+    sel_df = df[mask]
+
+    tpr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=1), 3))
+
+    mask = (df.label == "real")
+    sel_df = df[mask]
+
+    tnr_scores.append(round(recall_score(sel_df.gnd_truth.values, sel_df.pred.values, pos_label=0), 3))
+
+    sel_df = df.copy()
+    samples_tested.append(len(sel_df))
+    f1_scores.append(round(f1_score(sel_df.gnd_truth.values, sel_df.pred.values, average="macro"), 3))
+
+        
+    lb = pd.DataFrame({"Sample": columns + columns2 + ["overall (real + fake)"], "Num Samples": samples_tested, 
+                   "TPR": tpr_scores, "TNR": tnr_scores, "F1": f1_scores})
+    return lb
+    
 def build_leaderboard(results_path = 'results'):
   full_df = get_merged_df(results_path)
   full_df_mapped = map_df(full_df)
-  leaderboard = get_algorithm_scores(full_df_mapped)
+  leaderboard = get_algorithm_scores_v2(full_df_mapped)
   return leaderboard