import gradio as gr import pandas as pd import numpy as np from gradio_leaderboard import Leaderboard, SelectColumns, ColumnFilter TITLE = '''

TR-RewardBench: Evaluating Reward Models in Turkish

''' INTRODUCTION_TEXT = ''' Evaluating the chat, safety, reasoning, and translation capabilities of Reward Models in Turkish. This space is a more detailed version of [M-RewardBench](https://huggingface.co/spaces/C4AI-Community/m-rewardbench) for Turkish You can find more details (paper,code,etc.) on their space. This space uses the Turkish subset of [C4AI-Community/multilingual-reward-bench](https://hf.co/datasets/C4AI-Community/multilingual-reward-bench), I want to thank them for relasing this dataset 🤗. Most of the current models were evaluated with max token lenght of 2048. This effects the performance since it can cut some of the text. So if you try replicating the results with higher token size you may get slightly better results (which also depends on the model). Due to resource limit these results are just for a single run of each model. Running each model multiple times and taking the mean would give better representation of the actual performance. For the description of subsets you can check out the about section of the original [space](https://huggingface.co/spaces/allenai/reward-bench). ### Important warning ⚠️: In the original [English version of the dataset](https://huggingface.co/spaces/allenai/reward-bench) it is noted that some of the models are unintentionally contaminated. You can find more on [here](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300). I doubt that models can generalize enough to have a performance boost even if they are trained with the English translation of a dataset but I just wanted to warn anyways. ''' class AutoEvalColumn: model = { "name": "Model", "type": "markdown", "displayed_by_default": True, "never_hidden": True, } @classmethod def add_columns_from_df(cls, df, columns): for col in columns: if col.lower() != 'model': # Skip if it's the model column since it's predefined setattr(cls, col, { "name": col, "type": "markdown", "displayed_by_default": True, "never_hidden": False, }) class AutoEvalColumnCategorical: model = { "name": "Model", "type": "markdown", "displayed_by_default": True, "never_hidden": True, } @classmethod def add_columns_from_df(cls, df, columns): for col in columns: if col.lower() != 'model': # Skip if it's the model column since it's predefined setattr(cls, col, { "name": col, "type": "markdown", "displayed_by_default": True, "never_hidden": False, }) def get_result_data(): return pd.read_csv("model_performance.csv") def get_categorical_data(): return pd.read_csv("model_performance_categorical.csv") def init_leaderboard(dataframe, df_class): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[ col["type"] for col in df_class.__dict__.values() if isinstance(col, dict) ], select_columns=SelectColumns( default_selection=[ col["name"] for col in df_class.__dict__.values() if isinstance(col, dict) and col["displayed_by_default"] ], cant_deselect=[ col["name"] for col in df_class.__dict__.values() if isinstance(col, dict) and col.get("never_hidden", False) ], label="Select Columns to Display:", ), search_columns=["Model"], interactive=False, ) def format_model_link(row): """Format model name as HTML link if URL is available""" model_name = row["Model"] return model_name from functools import partial def format_with_color(val, min_val=50, max_val=100): """ Formats a value with inline green color gradient CSS. Returns an HTML string with bold, black text and muted green background. """ try: val = float(val) if pd.isna(val): return str(val) # Normalize value between 50 and 100 to 0-1 range normalized = (val - min_val) / (max_val - min_val) # Clamp value between 0 and 1 normalized = max(0, min(1, normalized)) # Create color gradient with reduced brightness (max 200 instead of 255) # and increased minimum intensity (50 instead of 0) intensity = int(50 + (150 * (1 - normalized))) # Return HTML with inline CSS - bold black text show_val = val*100 return f'
{show_val:.1f}
' except (ValueError, TypeError): return str(val) demo = gr.Blocks(theme=gr.themes.Soft()) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT) with gr.Tabs() as tabs: with gr.TabItem("🏅 Subset performance"): df = get_result_data() df = df.sort_values(by="accuracy", ascending=False) numeric_cols = df.select_dtypes(include=[np.number]).columns global_min = df.select_dtypes(include='number').min().min()#.astype(float) global_max = df.select_dtypes(include='number').max().max()#.astype(float) for col in numeric_cols: lang_format_with_color = partial(format_with_color, min_val=global_min, max_val=global_max, ) df[col] = df[col].apply(lang_format_with_color) AutoEvalColumn.add_columns_from_df(df, numeric_cols) leaderboard = init_leaderboard(df, AutoEvalColumn) with gr.TabItem("🏅 Categorical"): df = get_categorical_data() df = df.sort_values(by="Average", ascending=False) numeric_cols = df.select_dtypes(include=[np.number]).columns global_min = df.select_dtypes(include='number').min().min()#.astype(float) global_max = df.select_dtypes(include='number').max().max()#.astype(float) for col in numeric_cols: lang_format_with_color = partial(format_with_color, min_val=global_min, max_val=global_max, ) df[col] = df[col].apply(lang_format_with_color) AutoEvalColumnCategorical.add_columns_from_df(df, numeric_cols) leaderboard = init_leaderboard(df, AutoEvalColumnCategorical) with gr.Row(): with gr.Accordion("📚 Citation", open=False): citation_button = gr.Textbox( value=r""" @misc{kesim2024-tr-rewardbench, author = {Ege Kesim}, title = {TR-RewardBench: Evaluating Reward Models in Turkish}, year = {2024}, publisher = {Ege Kesim}, howpublished = "\url{https://huggingface.co/spaces/kesimeg/Turkish-rewardbench" } @misc{gureja2024mrewardbench, title={M-RewardBench: Evaluating Reward Models in Multilingual Settings}, author={Srishti Gureja and Lester James V. Miranda and Shayekh Bin Islam and Rishabh Maheshwary and Drishti Sharma and Gusti Winata and Nathan Lambert and Sebastian Ruder and Sara Hooker and Marzieh Fadaee}, year={2024}, eprint={2410.15522}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2410.15522}, }""", lines=7, label="BibTeX", elem_id="citation-button", show_copy_button=True, ) demo.launch(ssr_mode=False)