Merge pull request #1 from macrocosm-os/use-latest-scores

utils.py

@@ -1,4 +1,3 @@
-
 import os
 import math
 import time
@@ -137,16 +136,18 @@ def get_subnet_data(
     return result
 
 
-
-
 def get_wandb_runs(project: str, filters: Dict[str, Any]) -> List:
-    """Get the latest runs from Wandb, retrying infinitely until we get them."""
+    """Get the latest runs from Wandb, retrying infinitely until we get them.
+    
+    Returns:
+        List: List of runs matching the provided filters, newest run (by creation time) first."""
     while True:
         api = wandb.Api(api_key=WANDB_TOKEN)
         runs = list(
             api.runs(
                 project,
                 filters=filters,
+                order="-created_at",
             )
         )
         if len(runs) > 0:
@@ -160,6 +161,12 @@ def get_scores(
     uids: List[int],
     wandb_runs: List,
 ) -> Dict[int, Dict[str, Optional[float]]]:
+    """Returns the most recent scores for the provided UIDs.
+    
+    Args:
+        uids (List[int]): List of UIDs to get scores for.
+        wandb_runs (List): List of validator runs from Wandb. Requires the runs are provided in descending order.
+    """
     result = {}
     previous_timestamp = None
     # Iterate through the runs until we've processed all the uids.
@@ -170,9 +177,9 @@ def get_scores(
         all_uid_data = data["uid_data"]
         timestamp = data["timestamp"]
         # Make sure runs are indeed in descending time order.
-        #assert (
-        #previous_timestamp is None or timestamp < previous_timestamp
-        #), f"Timestamps are not in descending order: {timestamp} >= {previous_timestamp}"
+        assert (
+            previous_timestamp is None or timestamp < previous_timestamp
+        ), f"Timestamps are not in descending order: {timestamp} >= {previous_timestamp}"
         previous_timestamp = timestamp
 
         for uid in uids:
@@ -194,7 +201,6 @@ def get_scores(
     return result
 
 
-
 def get_validator_weights(
     metagraph: bt.metagraph,
 ) -> Dict[int, Tuple[float, int, Dict[int, float]]]:
@@ -214,8 +220,6 @@ def get_validator_weights(
     return ret
 
 
-
-
 def get_losses_over_time(wandb_runs: List) -> pd.DataFrame:
     """Returns a dataframe of the best average model loss over time."""
     timestamps = []
@@ -240,7 +244,6 @@ def get_losses_over_time(wandb_runs: List) -> pd.DataFrame:
     return pd.DataFrame({"timestamp": timestamps, "best_loss": best_losses})
 
 
-
 def next_epoch(subtensor: bt.subtensor, block: int) -> int:
     return (
         block
@@ -255,7 +258,6 @@ def is_floatable(x) -> bool:
     ) or isinstance(x, int)
 
 
-
 def format_score(uid: int, scores, key) -> Optional[float]:
     if uid in scores:
         if key in scores[uid]:
@@ -291,7 +293,7 @@ def get_benchmarks() -> Tuple[pd.DataFrame, datetime.datetime]:
         bt.logging.error("No benchmark project set.")
         return None, None
     runs = get_wandb_runs(project=BENCHMARK_WANDB_PROJECT, filters=None)
-    for run in runs[::-1]:
+    for run in runs:
         artifacts = list(run.logged_artifacts())
         if artifacts:
             table = artifacts[-1].get("benchmarks")