app.py

import gradio as gr
import pandas as pd
import numpy as np
from gradio_leaderboard import Leaderboard, SelectColumns, ColumnFilter


TITLE = '''<h1>
<span style="font-variant: small-caps;">TR-RewardBench</span>: Evaluating Reward Models in Turkish
</h1>'''
INTRODUCTION_TEXT = '''
Evaluating the chat, safety, reasoning, and translation capabilities of Reward Models in Turkish. This space is a more detailed version of [M-RewardBench](https://huggingface.co/spaces/C4AI-Community/m-rewardbench) for Turkish
You can find more details (paper,code,etc.) on their space. This space uses the Turkish subset of [C4AI-Community/multilingual-reward-bench](https://hf.co/datasets/C4AI-Community/multilingual-reward-bench), I want to thank them for relasing this dataset 🤗. 

Most of the current models were evaluated with max token lenght of 2048. This effects the performance since it can cut some of the text. So if you try replicating the results with higher token size
you may get slightly better results (which also depends on the model).

Due to resource limit these results are just for a single run of each model. Running each model multiple times and taking the mean would give better representation of the actual performance.

For the description of subsets you can check out the about section of the original [space](https://huggingface.co/spaces/allenai/reward-bench).

### Important warning ⚠️: 
In the original [English version of the dataset](https://huggingface.co/spaces/allenai/reward-bench) it is noted that some of the models are
unintentionally contaminated. You can find more on [here](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300). I doubt that models can generalize enough to have a performance boost even if they are trained with the English translation of a dataset but I just wanted to
warn anyways. 

'''

class AutoEvalColumn:
  model = {
    "name": "Model",
    "type": "markdown",
    "displayed_by_default": True,
    "never_hidden": True,
  }

  @classmethod
  def add_columns_from_df(cls, df, columns):
    for col in columns:
      if col.lower() != 'model':  # Skip if it's the model column since it's predefined
        setattr(cls, col, {
              "name": col,
              "type": "markdown",
              "displayed_by_default": True,
              "never_hidden": False,
        })				


class AutoEvalColumnCategorical:
  model = {
    "name": "Model",
    "type": "markdown",
    "displayed_by_default": True,
    "never_hidden": True,
  }

  @classmethod
  def add_columns_from_df(cls, df, columns):
    for col in columns:
      if col.lower() != 'model':  # Skip if it's the model column since it's predefined
        setattr(cls, col, {
              "name": col,
              "type": "markdown",
              "displayed_by_default": True,
              "never_hidden": False,
        })

def get_result_data():
  return pd.read_csv("model_performance.csv")


def get_categorical_data():
  return pd.read_csv("model_performance_categorical.csv")


def init_leaderboard(dataframe, df_class):
  if dataframe is None or dataframe.empty:
    raise ValueError("Leaderboard DataFrame is empty or None.")

  return Leaderboard(
    value=dataframe,
    datatype=[
      col["type"]
      for col in df_class.__dict__.values()
      if isinstance(col, dict)
    ],
    select_columns=SelectColumns(
      default_selection=[
        col["name"]
        for col in df_class.__dict__.values()
        if isinstance(col, dict) and col["displayed_by_default"]
      ],
      cant_deselect=[
        col["name"]
        for col in df_class.__dict__.values()
        if isinstance(col, dict) and col.get("never_hidden", False)
      ],
      label="Select Columns to Display:",
    ),
    search_columns=["Model"],
    interactive=False,
  )


def format_model_link(row):
  """Format model name as HTML link if URL is available"""
  model_name = row["Model"]

  return model_name

from functools import partial
def format_with_color(val, min_val=50, max_val=100):
    """
    Formats a value with inline green color gradient CSS.
    Returns an HTML string with bold, black text and muted green background.
    """
    try:
        val = float(val)
        if pd.isna(val):
            return str(val)
            
        # Normalize value between 50 and 100 to 0-1 range
        normalized = (val - min_val) / (max_val - min_val)

        # Clamp value between 0 and 1
        normalized = max(0, min(1, normalized))
        
        # Create color gradient with reduced brightness (max 200 instead of 255)
        # and increased minimum intensity (50 instead of 0)
        intensity = int(50 + (150 * (1 - normalized)))        
        
        # Return HTML with inline CSS - bold black text
        show_val = val*100
          
        return f'<div val={val} style="background-color: rgb({intensity}, 200, {intensity}); color: black; font-weight: bold; text-align: center; vertical-align: middle;">{show_val:.1f}</div>'
      
    except (ValueError, TypeError):
        return str(val)

demo = gr.Blocks(theme=gr.themes.Soft())

with demo:
  gr.HTML(TITLE)
  gr.Markdown(INTRODUCTION_TEXT)

  with gr.Tabs() as tabs:
    with gr.TabItem("🏅 Subset performance"):
      df = get_result_data()
      df = df.sort_values(by="accuracy", ascending=False)

      numeric_cols = df.select_dtypes(include=[np.number]).columns
      global_min = df.select_dtypes(include='number').min().min()#.astype(float)
      global_max = df.select_dtypes(include='number').max().max()#.astype(float)
      
      
      for col in numeric_cols:
        lang_format_with_color = partial(format_with_color, 
                                        min_val=global_min,
                                        max_val=global_max,
                                  )
    
        df[col] = df[col].apply(lang_format_with_color)
        

      AutoEvalColumn.add_columns_from_df(df, numeric_cols)
      leaderboard = init_leaderboard(df, AutoEvalColumn)
      
    with gr.TabItem("🏅 Categorical"):
      df = get_categorical_data()
      df = df.sort_values(by="Average", ascending=False)

      numeric_cols = df.select_dtypes(include=[np.number]).columns

      global_min = df.select_dtypes(include='number').min().min()#.astype(float)
      global_max = df.select_dtypes(include='number').max().max()#.astype(float)
      
      for col in numeric_cols:

        lang_format_with_color = partial(format_with_color, 
                                        min_val=global_min,
                                        max_val=global_max,
                                        )
        df[col] = df[col].apply(lang_format_with_color)
        
      
      AutoEvalColumnCategorical.add_columns_from_df(df, numeric_cols)
      leaderboard = init_leaderboard(df, AutoEvalColumnCategorical)
  
  with gr.Row():
    with gr.Accordion("📚 Citation", open=False):
      citation_button = gr.Textbox(
            value=r"""
            @misc{kesim2024-tr-rewardbench,
            author = {Ege Kesim},
            title = {TR-RewardBench: Evaluating Reward Models in Turkish},
            year = {2024},
            publisher = {Ege Kesim},
            howpublished = "\url{https://huggingface.co/spaces/kesimeg/Turkish-rewardbench"
            }           
            @misc{gureja2024mrewardbench,
          title={M-RewardBench: Evaluating Reward Models in Multilingual Settings}, 
          author={Srishti Gureja and Lester James V. Miranda and Shayekh Bin Islam and Rishabh Maheshwary and Drishti Sharma and Gusti Winata and Nathan Lambert and Sebastian Ruder and Sara Hooker and Marzieh Fadaee},
          year={2024},
          eprint={2410.15522},
          archivePrefix={arXiv},
          primaryClass={cs.CL},
          url={https://arxiv.org/abs/2410.15522}, 
          }""",
            lines=7,
            label="BibTeX",
            elem_id="citation-button",
            show_copy_button=True,
      )

demo.launch(ssr_mode=False)