Create app.py

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +0,0 @@
-.venv
-__pycache__/
-.env
-**.ipynb

api.py

@@ -1,135 +0,0 @@
-
-import atexit
-import datetime
-
-from flask import Flask, request, jsonify
-from apscheduler.schedulers.background import BackgroundScheduler
-
-import utils
-
-app = Flask(__name__)
-
-# Global variables (saves time on loading data)
-state_vars = None
-reload_timestamp = datetime.datetime.now().strftime('%D %T')
-
-
-def load_data(test=False):
-    """
-    Reload the state variables
-    """
-    global state_vars, reload_timestamp
-    if test:
-        state_vars = utils.test_load_state_vars()    
-    else:  
-        state_vars = utils.load_state_vars()
-        
-    reload_timestamp = datetime.datetime.now().strftime('%D %T')
-
-    print(f'Reloaded data at {reload_timestamp}')
-
-
-def start_scheduler():
-    scheduler = BackgroundScheduler()
-    scheduler.add_job(func=load_data, trigger="interval", seconds=60*30)
-    scheduler.start()
-
-    # Shut down the scheduler when exiting the app
-    atexit.register(lambda: scheduler.shutdown())
-
-
-@app.route('/', methods=['GET'])
-def home():
-    return "Welcome to the Bittensor Pretraining Leaderboard API!"
-
-
-@app.route('/updated', methods=['GET'])
-def updated():
-    return reload_timestamp
-
-
-@app.route('/benchmark', methods=['GET'])
-def benchmark():
-    """
-    Get the benchmarks and the timestamp
-
-    Returns:
-    - benchmarks: List of dicts (from pandas DataFrame)
-    - benchmark_timestamp: String
-    """
-
-    benchmarks = state_vars.get("benchmarks", None)
-    benchmark_timestamp = state_vars.get("benchmark_timestamp", None)
-
-    return jsonify(
-        {
-            "benchmarks": benchmarks.to_dict(orient='records'),
-            "benchmark_timestamp": benchmark_timestamp.strftime('%Y-%m-%d %H:%M:%S')
-        }
-    )
-
-@app.route('/metagraph', methods=['GET'])
-def metagraph():
-    """
-    Get the metagraph data
-    Returns:
-    - metagraph_data: List of dicts (from pandas DataFrame)
-    """
-    
-    metagraph = state_vars["metagraph"]
-     
-    return jsonify(
-        utils.make_metagraph_dataframe(metagraph).to_dict(orient='records')
-    )
-
-@app.route('/leaderboard', methods=['GET'])
-def leaderboard():
-    """
-    Get the leaderboard data
-    Returns:
-    - leaderboard_data: List of dicts (from pandas DataFrame)
-    """
-
-    model_data = state_vars["model_data"]
-    scores = state_vars["scores"]
-    show_stale = request.args.get('show_stale')
-    return jsonify(
-        utils.leaderboard_data(model_data, scores, show_stale=show_stale)
-        )
-
-
-@app.route('/loss', methods=['GET'])
-def loss():
-    """
-    Get the losses over time
-    Returns:
-    - losses_over_time: List of dicts (from pandas DataFrame)
-    """
-    vali_runs = state_vars["vali_runs"]
-
-    return jsonify(
-        utils.get_losses_over_time(vali_runs).to_dict(orient='records')
-        )
-
-
-@app.route('/validator', methods=['GET'])
-def validator():
-    """
-    Get the validator data
-    Returns:
-    - validator_data: List of dicts (from pandas DataFrame)
-    """
-    model_data = state_vars["model_data"]
-    validator_df = state_vars["validator_df"]       
-
-    return jsonify(
-        utils.make_validator_dataframe(validator_df, model_data).to_dict(orient='records')
-        )
-
-
-if __name__ == '__main__':
-    
-    load_data()
-    start_scheduler()
-
-    app.run(host='0.0.0.0', port=5000, debug=True)

app.py

@@ -1,191 +0,0 @@
-# Code adapted from: https://huggingface.co/spaces/RaoFoundation/pretraining-leaderboard/blob/main/app.py
-
-import datetime
-import os
-
-import gradio as gr
-import matplotlib.pyplot as plt
-from apscheduler.schedulers.background import BackgroundScheduler
-from dotenv import load_dotenv
-from huggingface_hub import HfApi
-
-import competitions
-import utils
-
-FONT = (
-    """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
-)
-TITLE = """<h1 align="center" id="space-title" class="typewriter">Finetuning Subnet Leaderboard</h1>"""
-HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/macrocosm-os/finetuning" target="_blank">Finetuning</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that rewards miners for producing finetuned models in defined competitions. The model with the best head-to-head score in each competition receive a steady emission of TAO.</h3>"""
-EVALUATION_HEADER = """<h3 align="center">Shows the latest per-competition evaluation statistics as calculated by the Taoverse validator</h3>"""
-
-HF_REPO_ID = "macrocosm-os/finetuning-leaderboard"
-SECONDS_PER_BLOCK = 12
-
-load_dotenv()
-
-HF_TOKEN = os.environ.get("HF_TOKEN", None)
-API = HfApi(token=HF_TOKEN)
-
-
-def get_next_update_div(current_block: int, next_update_block: int) -> str:
-    now = datetime.datetime.now()
-    blocks_to_go = next_update_block - current_block
-    next_update_time = now + datetime.timedelta(
-        seconds=blocks_to_go * SECONDS_PER_BLOCK
-    )
-    delta = next_update_time - now
-    return f"""<div align="center" style="font-size: larger;">Next reward update: <b>{blocks_to_go}</b> blocks (~{int(delta.total_seconds() // 60)} minutes)</div>"""
-
-
-def get_last_updated_div() -> str:
-    return f"""<div>Last Updated: {datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")} (UTC)</div>"""
-
-
-def restart_space():
-    API.restart_space(repo_id=HF_REPO_ID, token=HF_TOKEN)
-
-
-def main():
-    # To avoid leaderboard failures, infinitely try until we get all data
-    # needed to populate the dashboard
-
-    state_vars = utils.load_state_vars()
-    model_data = state_vars["model_data"]
-    vali_runs = state_vars["vali_runs"]
-    scores = state_vars["scores"]
-    validator_df = state_vars["validator_df"]
-    benchmarks_df = state_vars["benchmarks_df"]
-    benchmarks_targets = state_vars["benchmarks_targets"]
-
-    demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
-    with demo:
-        gr.HTML(FONT)
-        gr.HTML(TITLE)
-        gr.HTML(HEADER)
-
-        gr.Label(
-            label="Emissions",
-            value={
-                f"{c.namespace}/{c.name} ({c.commit[0:8]}) · (τ{round(c.emission, 2):,})": c.incentive
-                for c in model_data
-                if c.incentive
-            },
-            num_top_classes=10,
-        )
-
-        comp_ids = [2, 3]
-        with gr.Accordion("Competition Results"):
-            gr.HTML(EVALUATION_HEADER)
-            show_stale = gr.Checkbox(label="Show Stale", interactive=True)
-            competition_leaderboards = []
-            for comp_id in comp_ids:
-                details = competitions.COMPETITION_DETAILS[comp_id]
-                with gr.Accordion(f"{details.name} Competition"):
-                    gr.HTML(details.html_description)
-                    competition_leaderboards.append(
-                        gr.components.Dataframe(
-                            value=utils.leaderboard_data(
-                                model_data, scores, comp_id, show_stale.value
-                            ),
-                            headers=[
-                                "Name",
-                                "Win Rate",
-                                "Score",
-                                "Weight",
-                                "UID",
-                                "Block",
-                            ],
-                            datatype=[
-                                "markdown",
-                                "number",
-                                "number",
-                                "number",
-                                "number",
-                                "number",
-                            ],
-                            elem_id=f"comp{comp_id}-table",
-                            interactive=False,
-                            visible=True,
-                        )
-                    )
-            gr.HTML(
-                """
-                    <ul><li><b>Name:</b> the 🤗 Hugging Face repo (click to go to the model card)</li>
-                    <li><b>Win Rate:</b> % of head-to-head evals won vs. other eval'd models, given an epsilon advantage or disadvantage</li>
-                    <li><b>Score:</b> the combined model score as calculated by the OTF validator (lower is better)</li>
-                    <li><b>UID:</b> the Bittensor UID of the miner</li>
-                    <li><b>Weight:</b> the bittensor weight set for this model</li>
-                    <li><b>Block:</b> the Bittensor block that the model was submitted in</li></ul><br/>More stats on <a href="https://taostats.io/subnets/netuid-37/" target="_blank">taostats</a>."""
-            )
-            show_stale.change(
-                lambda stale: [
-                    utils.leaderboard_data(model_data, scores, id, stale)
-                    for id in comp_ids
-                ],
-                inputs=[show_stale],
-                outputs=competition_leaderboards,
-            )
-
-        if benchmarks_df is not None:
-
-            def create_benchmark_plot(benchmark: str, comp_id: int):
-                fig = plt.figure(figsize=(10, 8))
-
-                # Filter to just entries for this competition.
-                df = benchmarks_df[benchmarks_df["competition_id"] == comp_id]
-
-                plt.plot(df["timestamp"], df[benchmark])
-
-                # Adding horizontal dotted lines for various benchmark targets (well-known models)
-                for model, score in benchmarks_targets[benchmark].items():
-                    plt.axhline(y=score, linestyle="--", label=f"{model}")
-                    plt.text(
-                        benchmarks_df["timestamp"].max(),
-                        score,
-                        f"{model}",
-                        va="center",
-                        ha="right",
-                        backgroundcolor="white",
-                    )
-
-                # Adding labels and title
-                plt.ylabel(benchmark.upper())
-                plt.title(f"{benchmark.upper()} Over Time")
-                plt.xticks(rotation=45)
-
-                return fig
-
-            with gr.Accordion("Top Model Benchmarks"):
-                for comp_id in comp_ids:
-                    details = competitions.COMPETITION_DETAILS[comp_id]
-                    with gr.Accordion(f"{details.name} Benchmarks"):
-                        mmlu = create_benchmark_plot("mmlu", comp_id)
-                        mmlu_pro = create_benchmark_plot("mmlu_pro", comp_id)
-                        gr.Plot(mmlu)
-                        gr.Plot(mmlu_pro)
-                gr.HTML(
-                    """<div>Benchmarks computed using <a href='https://github.com/EleutherAI/lm-evaluation-harness'>lm-eval harness</a></div>"""
-                )
-                gr.HTML(
-                    """<ul><li>MMLU: Raw score</li><li>MMLU Pro: Normalized score using <a href='https://huggingface.co/docs/leaderboards/open_llm_leaderboard/normalization'>this</a> method</li></ul>"""
-                )
-
-        with gr.Accordion("Validator Stats"):
-            gr.components.Dataframe(
-                utils.make_validator_dataframe(validator_df, model_data),
-                interactive=False,
-                visible=True,
-            )
-        gr.HTML(value=get_last_updated_div())
-
-    scheduler = BackgroundScheduler()
-    scheduler.add_job(
-        restart_space, "interval", seconds=60 * 30
-    )  # restart every 15 minutes
-    scheduler.start()
-
-    demo.launch()
-
-
-main()

competitions.py

@@ -1,35 +0,0 @@
-from dataclasses import dataclass
-import html
-from typing import Dict
-
-
-@dataclass(frozen=True)
-class CompetitionDetails:
-    # The display name of the competition.
-    name: str
-
-    # The HTML description of the competition.
-    html_description: str
-
-
-# A map of competition IDs to HTML descriptions.
-COMPETITION_DETAILS: Dict[int, CompetitionDetails] = {
-    1: CompetitionDetails(
-        name="SN9_MODEL",
-        html_description="""<b>Competition ID 1</b><br/>Produce the best fine-tuned model from a Subnet 9 pretrained model. Models are evaluated using synthetic prompt/response data from Subnet 18.""",
-    ),
-    2: CompetitionDetails(
-        name="General Knowledge Chat-bot",
-        # TODO: Add link to SN1 dataset details.
-        html_description="""<b>Competition ID 2</b><br/>Produce the best general knowledge chat-bot. Models are evaluated using synthetic MMLU-like dataset from Subnet 1.""",
-    ),
-    3: CompetitionDetails(
-        name="General Knowledge Chat-bot (BYO tokenizer)",
-        html_description="""<b>Competition ID 3</b><br/>Produce the best general knowledge chat-bot. Models bring their own tokenizer and are evaluated using synthetic MMLU-like dataset from Subnet 1.""",
-    )
-}
-
-COMP_NAME_TO_ID = {
-    "B7_MULTI_CHOICE": 2,
-    "INSTRUCT_8B": 3,
-}

requirements.txt

@@ -1,12 +0,0 @@
-bittensor==7.3.1
-requests
-wandb==0.17.1
-numpy==1.26.4
-python-dotenv
-APScheduler
-huggingface-hub
-gradio
-pandas
-flask
-matplotlib
-

utils.py

@@ -1,505 +0,0 @@
-import argparse
-import datetime
-import functools
-import json
-import math
-import os
-import time
-import traceback
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
-
-import bittensor as bt
-import numpy as np
-import pandas as pd
-import wandb
-from bittensor.extrinsics.serving import get_metadata
-from dotenv import load_dotenv
-from wandb.apis.public.history import HistoryScan, SampledHistoryScan
-
-from competitions import COMP_NAME_TO_ID
-
-NETUID = 37
-DELAY_SECS = 3
-RETRIES = 3
-
-load_dotenv()
-
-WANDB_TOKEN = os.environ.get("WANDB_API_KEY", None)
-SUBTENSOR_ENDPOINT = os.environ.get("SUBTENSOR_ENDPOINT", None)
-VALIDATOR_WANDB_PROJECT = "rusticluftig/finetuning"
-BENCHMARK_WANDB_PROJECT = "rusticluftig/test-benchmarks"
-
-
-@dataclass(frozen=True)
-class ModelData:
-    uid: int
-    hotkey: str
-    competition_id: int
-    namespace: str
-    name: str
-    commit: str
-
-    # Hash of (hash(model) + hotkey)
-    secure_hash: str
-    block: int
-    incentive: float
-    emission: float
-
-    @classmethod
-    def from_compressed_str(
-        cls,
-        uid: int,
-        hotkey: str,
-        cs: str,
-        block: int,
-        incentive: float,
-        emission: float,
-    ):
-        """Returns an instance of this class from a compressed string representation"""
-        tokens = cs.split(":")
-        return ModelData(
-            uid=uid,
-            hotkey=hotkey,
-            namespace=tokens[0],
-            name=tokens[1],
-            commit=tokens[2],
-            secure_hash=tokens[3],
-            competition_id=int(tokens[4]),
-            block=block,
-            incentive=incentive,
-            emission=emission,
-        )
-
-
-def run_with_retries(func, *args, **kwargs):
-    """Runs a provided function with retries in the event of a failure."""
-    for i in range(0, RETRIES):
-        try:
-            return func(*args, **kwargs)
-        except (Exception, RuntimeError):
-            print(f"Failed to run function: {traceback.format_exc()}")
-            if i == RETRIES - 1:
-                raise
-            time.sleep(DELAY_SECS)
-    raise RuntimeError("Should never happen")
-
-
-def get_subtensor_and_metagraph() -> Tuple[bt.subtensor, bt.metagraph]:
-    """Returns a subtensor and metagraph for the finetuning subnet."""
-
-    def _internal() -> Tuple[bt.subtensor, bt.metagraph]:
-        if SUBTENSOR_ENDPOINT:
-            parser = argparse.ArgumentParser()
-            bt.subtensor.add_args(parser)
-            subtensor = bt.subtensor(
-                config=bt.config(
-                    parser=parser,
-                    args=["--subtensor.chain_endpoint", SUBTENSOR_ENDPOINT],
-                )
-            )
-        else:
-            subtensor = bt.subtensor("finney")
-
-        metagraph = subtensor.metagraph(NETUID, lite=False)
-
-        return subtensor, metagraph
-
-    return run_with_retries(_internal)
-
-
-def get_subnet_data(
-    subtensor: bt.subtensor, metagraph: bt.metagraph
-) -> List[ModelData]:
-    result = []
-    for uid in metagraph.uids.tolist():
-        hotkey = metagraph.hotkeys[uid]
-        metadata = None
-        try:
-            metadata = run_with_retries(
-                functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
-            )
-        except:
-            print(f"Failed to get metadata for UID {uid}: {traceback.format_exc()}")
-
-        if not metadata:
-            continue
-
-        commitment = metadata["info"]["fields"][0]
-        hex_data = commitment[list(commitment.keys())[0]][2:]
-        chain_str = bytes.fromhex(hex_data).decode()
-        block = metadata["block"]
-
-        incentive = np.nan_to_num(metagraph.incentive[uid]).item()
-        emission = (
-            np.nan_to_num(metagraph.emission[uid]).item() * 20
-        )  # convert to daily TAO
-
-        model_data = None
-        try:
-            model_data = ModelData.from_compressed_str(
-                uid, hotkey, chain_str, block, incentive, emission
-            )
-        except:
-            continue
-
-        result.append(model_data)
-    return result
-
-
-def get_wandb_runs(
-    project: str, filters: Dict[str, Any], order: str = "-created_at"
-) -> List:
-    """Get the latest runs from Wandb, retrying infinitely until we get them.
-
-    Args:
-        project (str): The Wandb project to get runs from.
-        filters (Dict[str, Any]): Filters to apply to the runs.
-        order (str): Order to sort the runs by. Defaults to "-created_at" (newest first)
-
-    Returns:
-        List: List of runs matching the provided filters
-    """
-    while True:
-        api = wandb.Api(api_key=WANDB_TOKEN, timeout=100)
-        runs = list(
-            api.runs(
-                project,
-                filters=filters,
-                order=order,
-            )
-        )
-        if len(runs) > 0:
-            return runs
-        # WandDB API is quite unreliable. Wait another minute and try again.
-        print("Failed to get runs from Wandb. Trying again in 60 seconds.")
-        time.sleep(60)
-
-
-def get_scores(
-    uids: List[int],
-    wandb_runs: List,
-) -> Dict[int, Dict[str, Optional[float]]]:
-    """Returns the most recent scores for the provided UIDs.
-
-    Args:
-        uids (List[int]): List of UIDs to get scores for.
-        wandb_runs (List): List of validator runs from Wandb. Requires the runs are provided in descending order.
-    """
-    result = {}
-    previous_timestamp = None
-    seen_competitions = set()
-    # Iterate through the runs until we've processed all the uids.
-    for i, run in enumerate(wandb_runs):
-        if not "original_format_json" in run.summary:
-            continue
-        data = json.loads(run.summary["original_format_json"])
-        all_uid_data = data["uid_data"]
-        timestamp = data["timestamp"]
-        # Make sure runs are indeed in descending time order.
-        assert (
-            previous_timestamp is None or timestamp < previous_timestamp
-        ), f"Timestamps are not in descending order: {timestamp} >= {previous_timestamp}"
-        previous_timestamp = timestamp
-
-        comp_id = data.get("competition_id", None)
-        for uid in uids:
-            if uid in result:
-                continue
-            if str(uid) in all_uid_data:
-                uid_data = all_uid_data[str(uid)]
-                # Only the most recent run per competition is fresh.
-                is_fresh = comp_id not in seen_competitions
-                result[uid] = {
-                    "avg_loss": uid_data.get("average_loss", None),
-                    "win_rate": uid_data.get("win_rate", None),
-                    "win_total": uid_data.get("win_total", None),
-                    "weight": uid_data.get("weight", None),
-                    "competition_id": uid_data.get("competition_id", None),
-                    "fresh": is_fresh,
-                }
-        seen_competitions.add(comp_id)
-        if len(result) == len(uids):
-            break
-    return result
-
-
-def get_validator_weights(
-    metagraph: bt.metagraph,
-) -> Dict[int, Tuple[float, int, Dict[int, float]]]:
-    """Returns a dictionary of validator UIDs to (vtrust, stake, {uid: weight})."""
-    ret = {}
-    for uid in metagraph.uids.tolist():
-        vtrust = metagraph.validator_trust[uid].item()
-        stake = metagraph.stake[uid].item()
-        if vtrust > 0 and stake > 10_000:
-            ret[uid] = (vtrust, stake, {})
-            for ouid in metagraph.uids.tolist():
-                if ouid == uid:
-                    continue
-                weight = round(metagraph.weights[uid][ouid].item(), 4)
-                if weight > 0:
-                    ret[uid][-1][ouid] = weight
-    return ret
-
-
-def get_losses_over_time(wandb_runs: List, competition_id: int) -> pd.DataFrame:
-    """Returns a dataframe of the best average model loss over time."""
-    timestamps = []
-    losses = []
-
-    for run in wandb_runs:
-        # For each run, check the 10 most recent steps.
-        best_loss = math.inf
-        should_add_datapoint = False
-        min_step = max(0, run.lastHistoryStep - 10)
-        history_scan = SampledHistoryScan(
-            run.client,
-            run,
-            ["original_format_json"],
-            min_step,
-            run.lastHistoryStep,
-            page_size=10,
-        )
-        max_timestamp = None
-        for step in history_scan:
-            data = json.loads(step["original_format_json"])
-            all_uid_data = data["uid_data"]
-            timestamp = datetime.datetime.fromtimestamp(data["timestamp"])
-            if max_timestamp is None:
-                max_timestamp = timestamp
-            max_timestamp = max(max_timestamp, timestamp)
-
-            for _, uid_data in all_uid_data.items():
-                loss = uid_data.get("average_loss", math.inf)
-                c_id = uid_data.get("competition_id", None)
-                if c_id is None or c_id != competition_id:
-                    continue
-
-                # Filter out issue caused by wandb unavailability.
-                if loss < 0.99 and loss < best_loss:
-                    best_loss = loss
-                    should_add_datapoint = True
-        # Now that we've processed the run's most recent steps, check if we should add a datapoint.
-        if should_add_datapoint:
-            timestamps.append(max_timestamp)
-            losses.append(best_loss)
-
-    return pd.DataFrame({"timestamp": timestamps, "losses": losses})
-
-
-def is_floatable(x) -> bool:
-    return (
-        isinstance(x, float) and not math.isnan(x) and not math.isinf(x)
-    ) or isinstance(x, int)
-
-
-def format_score(uid: int, scores, key) -> Optional[float]:
-    if uid in scores:
-        if key in scores[uid]:
-            point = scores[uid][key]
-            if is_floatable(point):
-                return round(scores[uid][key], 4)
-    return None
-
-
-def leaderboard_data(
-    leaderboard: List[ModelData],
-    scores: Dict[int, Dict[str, Optional[float]]],
-    competition_id: int,
-    show_stale: bool,
-) -> List[List[Any]]:
-    """Returns the leaderboard data, based on models data and UID scores."""
-    return [
-        [
-            f"[{c.namespace}/{c.name} ({c.commit[0:8]})](https://huggingface.co/{c.namespace}/{c.name}/commit/{c.commit})",
-            format_score(c.uid, scores, "win_rate"),
-            format_score(c.uid, scores, "avg_loss"),
-            format_score(c.uid, scores, "weight"),
-            c.uid,
-            c.block,
-        ]
-        for c in leaderboard
-        if c.competition_id == competition_id
-        and ((c.uid in scores and scores[c.uid]["fresh"]) or show_stale)
-    ]
-
-
-def get_benchmarks() -> Tuple[pd.DataFrame, Dict[str, Dict[str, float]]]:
-    """Returns the latest benchmarks and the time they were run."""
-    if not BENCHMARK_WANDB_PROJECT:
-        print("No benchmark project set.")
-        return None, None
-    runs = get_wandb_runs(
-        project=BENCHMARK_WANDB_PROJECT, filters=None, order="+created_at"
-    )
-    timestamps, uids, models, comp_ids, mmlu, mmlu_pro = [], [], [], [], [], []
-    for run in runs:
-        uid = run.config.get("uid", None)
-        model = run.config.get("model", None)
-        # Any run without a competition_id was for competition 2.
-        comp_name = run.config.get("competition_id", "B7_MULTI_CHOICE")
-        comp_id = COMP_NAME_TO_ID.get(comp_name, 2)
-        if not uid or not model:
-            continue
-        samples = list(
-            HistoryScan(
-                run.client,
-                run,
-                0,
-                1,
-            )
-        )
-        if not samples:
-            continue
-        sample = samples[0]
-        
-        # Make sure we have all the required keys.
-        has_all_keys = True
-        for required_key in ["mmlu.acc,none", "mmlu_pro", "_timestamp"]:
-            if required_key not in sample:
-                has_all_keys = False
-                break
-        if not has_all_keys:
-            continue
-        
-        comp_ids.append(comp_id)
-        timestamps.append(datetime.datetime.fromtimestamp(sample["_timestamp"]))
-        mmlu.append(sample["mmlu.acc,none"])
-        mmlu_pro.append(sample["mmlu_pro"])
-        uids.append(uid)
-        models.append(model)
-    return (
-        pd.DataFrame(
-            {
-                "timestamp": timestamps,
-                "uid": uids,
-                "model": models,
-                "competition_id": comp_ids,
-                "mmlu": mmlu,
-                "mmlu_pro": mmlu_pro,
-            }
-        ),
-        {
-            "mmlu": {
-                "Llama-3.1-8B-Instruct": 0.681,
-                "Mistral-7B-Instruct-v0.3": 0.597,
-                "gemma-2-9b-it": 0.719,
-            },
-            "mmlu_pro": {
-                "Llama-3.1-8B-Instruct": 30.68,
-                "Mistral-7B-Instruct-v0.3": 23.06,
-                "gemma-2-9b-it": 31.95,
-            },
-        },
-    )
-
-
-def make_validator_dataframe(
-    validator_df: pd.DataFrame, model_data: ModelData
-) -> pd.DataFrame:
-
-    values = [
-        [uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)]
-        + [validator_df[uid][-1].get(c.uid) for c in model_data if c.incentive]
-        for uid, _ in sorted(
-            zip(
-                validator_df.keys(),
-                [validator_df[x][1] for x in validator_df.keys()],
-            ),
-            key=lambda x: x[1],
-            reverse=True,
-        )
-    ]
-    dtypes = {"UID": int, "Stake (τ)": float, "V-Trust": float}
-    dtypes.update(
-        {
-            f"{c.namespace}/{c.name} ({c.commit[0:8]})": float
-            for c in model_data
-            if c.incentive
-        }
-    )
-    return pd.DataFrame(values, columns=dtypes.keys()).astype(dtypes)
-
-
-def make_metagraph_dataframe(metagraph: bt.metagraph, weights=False) -> pd.DataFrame:
-
-    cols = [
-        "stake",
-        "emission",
-        "trust",
-        "validator_trust",
-        "dividends",
-        "incentive",
-        "R",
-        "consensus",
-        "validator_permit",
-    ]
-
-    frame = pd.DataFrame({k: getattr(metagraph, k) for k in cols})
-    frame["block"] = metagraph.block.item()
-    frame["netuid"] = NETUID
-    frame["uid"] = range(len(frame))
-    frame["hotkey"] = [axon.hotkey for axon in metagraph.axons]
-    frame["coldkey"] = [axon.coldkey for axon in metagraph.axons]
-    if weights and metagraph.W is not None:
-        # convert NxN tensor to a list of lists so it fits into the dataframe
-        frame["weights"] = [w.tolist() for w in metagraph.W]
-
-    return frame
-
-
-def load_state_vars() -> dict[Any]:
-    while True:
-        try:
-            subtensor, metagraph = get_subtensor_and_metagraph()
-
-            print(f"Loaded subtensor and metagraph: {metagraph}")
-
-            model_data: List[ModelData] = get_subnet_data(subtensor, metagraph)
-            model_data.sort(key=lambda x: x.incentive, reverse=True)
-            print(f"Loaded {len(model_data)} models")
-
-            vali_runs = get_wandb_runs(
-                project=VALIDATOR_WANDB_PROJECT,
-                filters={
-                    "$and": [{"config.type": "validator"}],
-                    "$or": [{"config.uid": 28}, {"config.uid": 16}],
-                },
-            )
-            print(f"Loaded {len(vali_runs)} validator runs")
-
-            scores = get_scores([x.uid for x in model_data], vali_runs)
-            print(f"Loaded {len(scores)} scores")
-
-            validator_df = get_validator_weights(metagraph)
-            weight_keys = set()
-            for uid, stats in validator_df.items():
-                weight_keys.update(stats[-1].keys())
-            print("Loaded validator weights")
-
-            # Compute loss over time for all competitions.
-            # losses_2 = get_losses_over_time(vali_runs, 2)
-            # print("Loaded losses over time for comp 2")
-
-            benchmarks_df, benchmarks_targets = get_benchmarks()
-            print("Loaded benchmarks")
-            break
-
-        except KeyboardInterrupt:
-            print("Exiting...")
-            break
-
-        except Exception as e:
-            print(f"Failed to get data: {traceback.format_exc()}")
-            time.sleep(30)
-
-    return {
-        "metagraph": metagraph,
-        "model_data": model_data,
-        "vali_runs": vali_runs,
-        "scores": scores,
-        "validator_df": validator_df,
-        "benchmarks_df": benchmarks_df,
-        "benchmarks_targets": benchmarks_targets,
-    }