Spaces:
Running
Running
Enabled getting leaderboard and model tournament table and CSV for pre-submit
Browse files
server.py
CHANGED
|
@@ -436,30 +436,47 @@ class LeaderboardServer:
|
|
| 436 |
df_css[c].iloc[i] = ''
|
| 437 |
return df_css
|
| 438 |
|
| 439 |
-
def get_model_tournament_table_csv(self, submission_id, category):
|
| 440 |
-
|
| 441 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
|
| 443 |
-
def get_model_tournament_table(self, submission_id, category):
|
| 444 |
-
|
| 445 |
-
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
-
def _get_model_tournament_table(self, submission_id, category, to_csv=False):
|
| 448 |
model_tournament_table = []
|
| 449 |
|
| 450 |
with self.var_lock.ro:
|
| 451 |
-
|
| 452 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 453 |
|
| 454 |
match_results = {}
|
| 455 |
-
for task in
|
| 456 |
task_category = self.TASKS_METADATA[task]["category"]
|
| 457 |
if category in (task_category, self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
|
| 458 |
if to_csv:
|
| 459 |
-
match_results[task] =
|
| 460 |
else:
|
| 461 |
match_task_result_details = dict.fromkeys(["significant", "p_value"]) # order has impact to sorting DataFrame
|
| 462 |
-
match_task_result_details.update(copy.deepcopy(
|
| 463 |
match_task_result_details["significant"] = str(match_task_result_details["significant"]).lower() # originaly bool
|
| 464 |
match_task_result_significant = match_task_result_details["significant"]
|
| 465 |
match_task_result_details = "\n".join(f"{k}: {v}" for k, v in match_task_result_details.items())
|
|
@@ -520,10 +537,16 @@ class LeaderboardServer:
|
|
| 520 |
dataframe.to_csv(filepath, index=False)
|
| 521 |
return filepath
|
| 522 |
|
| 523 |
-
def get_leaderboard_csv(self, category=None):
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 527 |
|
| 528 |
def get_leaderboard(self, pre_submit=None, category=None):
|
| 529 |
if pre_submit == None:
|
|
|
|
| 436 |
df_css[c].iloc[i] = ''
|
| 437 |
return df_css
|
| 438 |
|
| 439 |
+
def get_model_tournament_table_csv(self, submission_id, category, pre_submit=None):
|
| 440 |
+
if pre_submit == None:
|
| 441 |
+
with self.var_lock.ro:
|
| 442 |
+
return self.tournament_dataframes_csv[submission_id][category]
|
| 443 |
+
else:
|
| 444 |
+
return self._dataframe_to_csv(
|
| 445 |
+
self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit, to_csv=True),
|
| 446 |
+
f"Tournament table - pre-submit - {category}.csv",
|
| 447 |
+
)
|
| 448 |
|
| 449 |
+
def get_model_tournament_table(self, submission_id, category, pre_submit=None):
|
| 450 |
+
if pre_submit == None:
|
| 451 |
+
with self.var_lock.ro:
|
| 452 |
+
return copy.copy(self.tournament_dataframes[submission_id][category])
|
| 453 |
+
else:
|
| 454 |
+
return self._get_model_tournament_table(submission_id, category, pre_submit=pre_submit)
|
| 455 |
|
| 456 |
+
def _get_model_tournament_table(self, submission_id, category, pre_submit=None, to_csv=False):
|
| 457 |
model_tournament_table = []
|
| 458 |
|
| 459 |
with self.var_lock.ro:
|
| 460 |
+
tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
|
| 461 |
+
|
| 462 |
+
for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
|
| 463 |
+
if competitor_id not in self.submission_id_to_data:
|
| 464 |
+
if pre_submit and competitor_id == pre_submit.submission_id:
|
| 465 |
+
data = json.load(open(pre_submit.file))
|
| 466 |
+
else:
|
| 467 |
+
raise gr.Error(f"Internal error: Submission [{competitor_id}] not found")
|
| 468 |
+
else:
|
| 469 |
+
data = self.submission_id_to_data[competitor_id]
|
| 470 |
|
| 471 |
match_results = {}
|
| 472 |
+
for task in tournament_results[submission_id][competitor_id]:
|
| 473 |
task_category = self.TASKS_METADATA[task]["category"]
|
| 474 |
if category in (task_category, self.TASKS_CATEGORY_OVERALL, self.TASKS_CATEGORY_OVERALL_DETAILS):
|
| 475 |
if to_csv:
|
| 476 |
+
match_results[task] = tournament_results[submission_id][competitor_id][task]["significant"]
|
| 477 |
else:
|
| 478 |
match_task_result_details = dict.fromkeys(["significant", "p_value"]) # order has impact to sorting DataFrame
|
| 479 |
+
match_task_result_details.update(copy.deepcopy(tournament_results[submission_id][competitor_id][task]))
|
| 480 |
match_task_result_details["significant"] = str(match_task_result_details["significant"]).lower() # originaly bool
|
| 481 |
match_task_result_significant = match_task_result_details["significant"]
|
| 482 |
match_task_result_details = "\n".join(f"{k}: {v}" for k, v in match_task_result_details.items())
|
|
|
|
| 537 |
dataframe.to_csv(filepath, index=False)
|
| 538 |
return filepath
|
| 539 |
|
| 540 |
+
def get_leaderboard_csv(self, pre_submit=None, category=None):
|
| 541 |
+
if pre_submit == None:
|
| 542 |
+
category = category if category else self.TASKS_CATEGORY_OVERALL
|
| 543 |
+
with self.var_lock.ro:
|
| 544 |
+
return self.leaderboard_dataframes_csv[category]
|
| 545 |
+
else:
|
| 546 |
+
return self._dataframe_to_csv(
|
| 547 |
+
self._get_leaderboard(pre_submit=pre_submit, category=category, to_csv=True),
|
| 548 |
+
f"Leaderboard - pre-submit - {category}.csv",
|
| 549 |
+
)
|
| 550 |
|
| 551 |
def get_leaderboard(self, pre_submit=None, category=None):
|
| 552 |
if pre_submit == None:
|