fix filtering
Browse files- app.py +81 -68
- src/build.py +16 -15
- src/utils.py +68 -10
app.py
CHANGED
|
@@ -1,65 +1,39 @@
|
|
|
|
|
| 1 |
import gradio as gr
|
| 2 |
import pandas as pd
|
| 3 |
-
|
| 4 |
-
|
| 5 |
|
| 6 |
df = pd.read_csv("data/code_eval_board.csv")
|
| 7 |
|
| 8 |
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
|
| 9 |
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
|
| 10 |
-
COLS_LITE = [
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
y=[df.loc[i, 'Average score']],
|
| 30 |
-
mode='markers',
|
| 31 |
-
marker=dict(
|
| 32 |
-
size=[df.loc[i, 'Size (B)'] + 10],
|
| 33 |
-
color=df.loc[i, 'color'],
|
| 34 |
-
symbol=df.loc[i, 'symbol']
|
| 35 |
-
),
|
| 36 |
-
name=df.loc[i, 'Models'],
|
| 37 |
-
hovertemplate =
|
| 38 |
-
'<b>%{text}</b><br><br>' +
|
| 39 |
-
f'{throughput_column}: %{{x}}<br>'+
|
| 40 |
-
'Average Score: %{y}<br>' +
|
| 41 |
-
'Peak Memory (MB): ' + str(df.loc[i, 'Peak Memory (MB)']) + '<br>' +
|
| 42 |
-
'Human Eval (Python): ' + str(df.loc[i, 'humaneval-python']),
|
| 43 |
-
text=[df.loc[i, 'Models']],
|
| 44 |
-
showlegend=True
|
| 45 |
-
))
|
| 46 |
-
|
| 47 |
-
fig.update_layout(
|
| 48 |
-
autosize=False,
|
| 49 |
-
width=650,
|
| 50 |
-
height=600,
|
| 51 |
-
title=f'Average Score Vs Throughput (A100-80GB, Float16, Batch Size <b>{bs}</b>)',
|
| 52 |
-
xaxis_title=f'{throughput_column}',
|
| 53 |
-
yaxis_title='Average Code Score',
|
| 54 |
-
)
|
| 55 |
-
return fig
|
| 56 |
|
| 57 |
|
| 58 |
def filter_items(df, leaderboard_table, query):
|
| 59 |
if query == "all":
|
| 60 |
return df[leaderboard_table.columns]
|
| 61 |
else:
|
| 62 |
-
query = query[0]
|
| 63 |
filtered_df = df[(df["T"] == query)]
|
| 64 |
return filtered_df[leaderboard_table.columns]
|
| 65 |
|
|
@@ -87,12 +61,30 @@ with demo:
|
|
| 87 |
with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
|
| 88 |
with gr.TabItem("🔍 Evaluation table", id=0):
|
| 89 |
with gr.Column():
|
| 90 |
-
#with gr.Column(min_width=780):
|
| 91 |
shown_columns = gr.CheckboxGroup(
|
| 92 |
-
choices
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
interactive=True,
|
| 97 |
)
|
| 98 |
with gr.Row():
|
|
@@ -103,38 +95,59 @@ with demo:
|
|
| 103 |
)
|
| 104 |
filter_columns = gr.Radio(
|
| 105 |
label="⏚ Filter model types",
|
| 106 |
-
choices
|
| 107 |
value="all",
|
| 108 |
-
elem_id="filter-columns"
|
| 109 |
)
|
| 110 |
-
|
| 111 |
-
# value=df, headers=COLS, datatype=["str" for _ in range(len(COLS))]
|
| 112 |
-
#)
|
| 113 |
leaderboard_df = gr.components.Dataframe(
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
hidden_leaderboard_df = gr.components.Dataframe(
|
| 120 |
-
value=df,
|
|
|
|
|
|
|
|
|
|
| 121 |
)
|
| 122 |
search_bar.submit(
|
| 123 |
search_table,
|
| 124 |
[hidden_leaderboard_df, leaderboard_df, search_bar],
|
| 125 |
leaderboard_df,
|
| 126 |
)
|
| 127 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
|
| 129 |
with gr.TabItem("📊 Performance Plot", id=1):
|
| 130 |
with gr.Row():
|
| 131 |
bs_1_plot = gr.components.Plot(
|
| 132 |
-
value=plot_throughput(bs=1),
|
| 133 |
elem_id="bs1-plot",
|
| 134 |
show_label=False,
|
| 135 |
)
|
| 136 |
bs_50_plt = gr.components.Plot(
|
| 137 |
-
value=plot_throughput(bs=50),
|
| 138 |
elem_id="bs50-plot",
|
| 139 |
show_label=False,
|
| 140 |
)
|
|
|
|
| 1 |
+
# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
|
| 2 |
import gradio as gr
|
| 3 |
import pandas as pd
|
| 4 |
+
from src.utils import AutoEvalColumn, fields, make_clickable_names, plot_throughput
|
| 5 |
+
|
| 6 |
|
| 7 |
df = pd.read_csv("data/code_eval_board.csv")
|
| 8 |
|
| 9 |
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
|
| 10 |
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
|
| 11 |
+
COLS_LITE = [
|
| 12 |
+
c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
|
| 13 |
+
]
|
| 14 |
+
TYPES_LITE = [
|
| 15 |
+
c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def select_columns(df, columns):
|
| 20 |
+
always_here_cols = [
|
| 21 |
+
AutoEvalColumn.model_type_symbol.name,
|
| 22 |
+
AutoEvalColumn.model.name,
|
| 23 |
+
]
|
| 24 |
+
# We use COLS to maintain sorting
|
| 25 |
+
filtered_df = df[
|
| 26 |
+
always_here_cols
|
| 27 |
+
+ [c for c in COLS if c in df.columns and c in columns]
|
| 28 |
+
]
|
| 29 |
+
return filtered_df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def filter_items(df, leaderboard_table, query):
|
| 33 |
if query == "all":
|
| 34 |
return df[leaderboard_table.columns]
|
| 35 |
else:
|
| 36 |
+
query = query[0] # take only the emoji character
|
| 37 |
filtered_df = df[(df["T"] == query)]
|
| 38 |
return filtered_df[leaderboard_table.columns]
|
| 39 |
|
|
|
|
| 61 |
with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
|
| 62 |
with gr.TabItem("🔍 Evaluation table", id=0):
|
| 63 |
with gr.Column():
|
| 64 |
+
# with gr.Column(min_width=780):
|
| 65 |
shown_columns = gr.CheckboxGroup(
|
| 66 |
+
choices=[
|
| 67 |
+
c
|
| 68 |
+
for c in COLS
|
| 69 |
+
if c
|
| 70 |
+
not in [
|
| 71 |
+
AutoEvalColumn.dummy.name,
|
| 72 |
+
AutoEvalColumn.model.name,
|
| 73 |
+
AutoEvalColumn.model_type_symbol.name,
|
| 74 |
+
]
|
| 75 |
+
],
|
| 76 |
+
value=[
|
| 77 |
+
c
|
| 78 |
+
for c in COLS_LITE
|
| 79 |
+
if c
|
| 80 |
+
not in [
|
| 81 |
+
AutoEvalColumn.dummy.name,
|
| 82 |
+
AutoEvalColumn.model.name,
|
| 83 |
+
AutoEvalColumn.model_type_symbol.name,
|
| 84 |
+
]
|
| 85 |
+
],
|
| 86 |
+
label="Select columns to show",
|
| 87 |
+
elem_id="column-select",
|
| 88 |
interactive=True,
|
| 89 |
)
|
| 90 |
with gr.Row():
|
|
|
|
| 95 |
)
|
| 96 |
filter_columns = gr.Radio(
|
| 97 |
label="⏚ Filter model types",
|
| 98 |
+
choices=["all", "🟢 base", "🔶 instruction-tuned"],
|
| 99 |
value="all",
|
| 100 |
+
elem_id="filter-columns",
|
| 101 |
)
|
| 102 |
+
|
|
|
|
|
|
|
| 103 |
leaderboard_df = gr.components.Dataframe(
|
| 104 |
+
value=df[
|
| 105 |
+
[
|
| 106 |
+
AutoEvalColumn.model_type_symbol.name,
|
| 107 |
+
AutoEvalColumn.model.name,
|
| 108 |
+
]
|
| 109 |
+
+ shown_columns.value
|
| 110 |
+
],
|
| 111 |
+
headers=[
|
| 112 |
+
AutoEvalColumn.model_type_symbol.name,
|
| 113 |
+
AutoEvalColumn.model.name,
|
| 114 |
+
]
|
| 115 |
+
+ shown_columns.value,
|
| 116 |
+
datatype=TYPES,
|
| 117 |
+
elem_id="leaderboard-table",
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
hidden_leaderboard_df = gr.components.Dataframe(
|
| 121 |
+
value=df,
|
| 122 |
+
headers=COLS,
|
| 123 |
+
datatype=["str" for _ in range(len(COLS))],
|
| 124 |
+
visible=False,
|
| 125 |
)
|
| 126 |
search_bar.submit(
|
| 127 |
search_table,
|
| 128 |
[hidden_leaderboard_df, leaderboard_df, search_bar],
|
| 129 |
leaderboard_df,
|
| 130 |
)
|
| 131 |
+
shown_columns.change(
|
| 132 |
+
select_columns,
|
| 133 |
+
[hidden_leaderboard_df, shown_columns],
|
| 134 |
+
leaderboard_df,
|
| 135 |
+
)
|
| 136 |
+
filter_columns.change(
|
| 137 |
+
filter_items,
|
| 138 |
+
[hidden_leaderboard_df, leaderboard_df, filter_columns],
|
| 139 |
+
leaderboard_df,
|
| 140 |
+
)
|
| 141 |
|
| 142 |
with gr.TabItem("📊 Performance Plot", id=1):
|
| 143 |
with gr.Row():
|
| 144 |
bs_1_plot = gr.components.Plot(
|
| 145 |
+
value=plot_throughput(df, bs=1),
|
| 146 |
elem_id="bs1-plot",
|
| 147 |
show_label=False,
|
| 148 |
)
|
| 149 |
bs_50_plt = gr.components.Plot(
|
| 150 |
+
value=plot_throughput(df, bs=50),
|
| 151 |
elem_id="bs50-plot",
|
| 152 |
show_label=False,
|
| 153 |
)
|
src/build.py
CHANGED
|
@@ -21,25 +21,26 @@ df = df[["Models", "Size (B)", "Win Rate"] + df.columns[2:-1].tolist()]
|
|
| 21 |
# sort with regard to column win rate
|
| 22 |
df = df.sort_values(by=["Win Rate"], ascending=False)
|
| 23 |
# add column with model links as https://huggingface.co/WizardLM/WizardCoder-15B-V1.0, https://huggingface.co/bigcode/starcoder, https://huggingface.co/bigcode/starcoderbase, https://huggingface.co/bigcode/starcoderbase-7b,
|
| 24 |
-
#https://huggingface.co/bigcode/starcoderbase-3b, https://huggingface.co/bigcode/starcoderbase-1b, https://huggingface.co/bigcode/santacoder, https://huggingface.co/replit/replit-code-v1-3b, https://huggingface.co/THUDM/codegeex2-6b
|
| 25 |
|
| 26 |
-
links = {
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
|
|
|
| 39 |
df["Links"] = df["Models"].map(links)
|
| 40 |
|
| 41 |
df.insert(0, "T", "🟢")
|
| 42 |
df.loc[df["Models"].str.contains("WizardCoder"), "T"] = "🔶"
|
| 43 |
print(df)
|
| 44 |
df.to_csv("data/code_eval_board.csv", index=False)
|
| 45 |
-
# print first 10 cols
|
|
|
|
| 21 |
# sort with regard to column win rate
|
| 22 |
df = df.sort_values(by=["Win Rate"], ascending=False)
|
| 23 |
# add column with model links as https://huggingface.co/WizardLM/WizardCoder-15B-V1.0, https://huggingface.co/bigcode/starcoder, https://huggingface.co/bigcode/starcoderbase, https://huggingface.co/bigcode/starcoderbase-7b,
|
| 24 |
+
# https://huggingface.co/bigcode/starcoderbase-3b, https://huggingface.co/bigcode/starcoderbase-1b, https://huggingface.co/bigcode/santacoder, https://huggingface.co/replit/replit-code-v1-3b, https://huggingface.co/THUDM/codegeex2-6b
|
| 25 |
|
| 26 |
+
links = {
|
| 27 |
+
"WizardCoder-15B-V1.0": "https://huggingface.co/WizardLM/WizardCoder-15B-V1.0",
|
| 28 |
+
"StarCoder-15B": "https://huggingface.co/bigcode/starcoder",
|
| 29 |
+
"StarCoderBase-15B": "https://huggingface.co/bigcode/starcoderbase",
|
| 30 |
+
"StarCoderBase-7B": "https://huggingface.co/bigcode/starcoderbase-7b",
|
| 31 |
+
"StarCoderBase-3B": "https://huggingface.co/bigcode/starcoderbase-3b",
|
| 32 |
+
"StarCoderBase-1.1B": "https://huggingface.co/bigcode/starcoderbase-1b",
|
| 33 |
+
"SantaCoder-1.1B": "https://huggingface.co/bigcode/santacoder",
|
| 34 |
+
"Replit-2.7B": "https://huggingface.co/replit/replit-code-v1-3b",
|
| 35 |
+
"CodeGeex2-6B": "https://huggingface.co/THUDM/codegeex2-6b",
|
| 36 |
+
"CodeGen25-7B-multi": "https://huggingface.co/Salesforce/codegen25-7b-multi",
|
| 37 |
+
"CodeGen25-7B-mono": "https://huggingface.co/Salesforce/codegen25-7b-mono",
|
| 38 |
+
"CodeGen-16B-Multi": "https://huggingface.co/Salesforce/codegen-16B-multi",
|
| 39 |
+
}
|
| 40 |
df["Links"] = df["Models"].map(links)
|
| 41 |
|
| 42 |
df.insert(0, "T", "🟢")
|
| 43 |
df.loc[df["Models"].str.contains("WizardCoder"), "T"] = "🔶"
|
| 44 |
print(df)
|
| 45 |
df.to_csv("data/code_eval_board.csv", index=False)
|
| 46 |
+
# print first 10 cols
|
src/utils.py
CHANGED
|
@@ -1,20 +1,25 @@
|
|
| 1 |
-
#source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
|
| 2 |
from dataclasses import dataclass
|
|
|
|
| 3 |
|
| 4 |
-
# These classes are for user facing column names, to avoid having to change them
|
| 5 |
-
# all around the code when a modif is needed
|
| 6 |
@dataclass
|
| 7 |
class ColumnContent:
|
| 8 |
name: str
|
| 9 |
-
type: str
|
| 10 |
-
displayed_by_default: bool
|
| 11 |
hidden: bool = False
|
| 12 |
|
|
|
|
| 13 |
def fields(raw_class):
|
| 14 |
-
return [
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
@dataclass(frozen=True)
|
| 17 |
-
class AutoEvalColumn:
|
| 18 |
model_type_symbol = ColumnContent("T", "str", True)
|
| 19 |
model = ColumnContent("Models", "markdown", True)
|
| 20 |
win_rate = ColumnContent("Win Rate", "number", True)
|
|
@@ -35,7 +40,7 @@ class AutoEvalColumn: # Auto evals column
|
|
| 35 |
throughput_bs50 = ColumnContent("Throughput (tokens/s) bs=50", "number", False)
|
| 36 |
peak_memory = ColumnContent("Peak Memory (MB)", "number", False)
|
| 37 |
seq_length = ColumnContent("Seq_length", "number", False)
|
| 38 |
-
average = ColumnContent("Average
|
| 39 |
link = ColumnContent("Links", "str", False)
|
| 40 |
dummy = ColumnContent("Models", "str", False)
|
| 41 |
|
|
@@ -45,5 +50,58 @@ def model_hyperlink(link, model_name):
|
|
| 45 |
|
| 46 |
|
| 47 |
def make_clickable_names(df):
|
| 48 |
-
df[
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
|
| 2 |
from dataclasses import dataclass
|
| 3 |
+
import plotly.graph_objects as go
|
| 4 |
|
| 5 |
+
# These classes are for user facing column names, to avoid having to change them
|
| 6 |
+
# all around the code when a modif is needed
|
| 7 |
@dataclass
|
| 8 |
class ColumnContent:
|
| 9 |
name: str
|
| 10 |
+
type: str
|
| 11 |
+
displayed_by_default: bool
|
| 12 |
hidden: bool = False
|
| 13 |
|
| 14 |
+
|
| 15 |
def fields(raw_class):
|
| 16 |
+
return [
|
| 17 |
+
v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
|
| 18 |
+
]
|
| 19 |
+
|
| 20 |
|
| 21 |
@dataclass(frozen=True)
|
| 22 |
+
class AutoEvalColumn: # Auto evals column
|
| 23 |
model_type_symbol = ColumnContent("T", "str", True)
|
| 24 |
model = ColumnContent("Models", "markdown", True)
|
| 25 |
win_rate = ColumnContent("Win Rate", "number", True)
|
|
|
|
| 40 |
throughput_bs50 = ColumnContent("Throughput (tokens/s) bs=50", "number", False)
|
| 41 |
peak_memory = ColumnContent("Peak Memory (MB)", "number", False)
|
| 42 |
seq_length = ColumnContent("Seq_length", "number", False)
|
| 43 |
+
average = ColumnContent("Average score", "number", False)
|
| 44 |
link = ColumnContent("Links", "str", False)
|
| 45 |
dummy = ColumnContent("Models", "str", False)
|
| 46 |
|
|
|
|
| 50 |
|
| 51 |
|
| 52 |
def make_clickable_names(df):
|
| 53 |
+
df["Models"] = df.apply(
|
| 54 |
+
lambda row: model_hyperlink(row["Links"], row["Models"]), axis=1
|
| 55 |
+
)
|
| 56 |
+
return df
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def plot_throughput(df, bs=1):
|
| 60 |
+
throughput_column = (
|
| 61 |
+
"Throughput (tokens/s)" if bs == 1 else "Throughput (tokens/s) bs=50"
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
df["symbol"] = 2 # Triangle
|
| 65 |
+
df["color"] = ""
|
| 66 |
+
df.loc[df["Models"].str.contains("StarCoder|SantaCoder"), "color"] = "orange"
|
| 67 |
+
df.loc[df["Models"].str.contains("CodeGen"), "color"] = "pink"
|
| 68 |
+
df.loc[df["Models"].str.contains("Replit"), "color"] = "purple"
|
| 69 |
+
df.loc[df["Models"].str.contains("WizardCoder"), "color"] = "green"
|
| 70 |
+
df.loc[df["Models"].str.contains("CodeGeex"), "color"] = "blue"
|
| 71 |
+
|
| 72 |
+
fig = go.Figure()
|
| 73 |
+
|
| 74 |
+
for i in df.index:
|
| 75 |
+
fig.add_trace(
|
| 76 |
+
go.Scatter(
|
| 77 |
+
x=[df.loc[i, throughput_column]],
|
| 78 |
+
y=[df.loc[i, "Average score"]],
|
| 79 |
+
mode="markers",
|
| 80 |
+
marker=dict(
|
| 81 |
+
size=[df.loc[i, "Size (B)"] + 10],
|
| 82 |
+
color=df.loc[i, "color"],
|
| 83 |
+
symbol=df.loc[i, "symbol"],
|
| 84 |
+
),
|
| 85 |
+
name=df.loc[i, "Models"],
|
| 86 |
+
hovertemplate="<b>%{text}</b><br><br>"
|
| 87 |
+
+ f"{throughput_column}: %{{x}}<br>"
|
| 88 |
+
+ "Average Score: %{y}<br>"
|
| 89 |
+
+ "Peak Memory (MB): "
|
| 90 |
+
+ str(df.loc[i, "Peak Memory (MB)"])
|
| 91 |
+
+ "<br>"
|
| 92 |
+
+ "Human Eval (Python): "
|
| 93 |
+
+ str(df.loc[i, "humaneval-python"]),
|
| 94 |
+
text=[df.loc[i, "Models"]],
|
| 95 |
+
showlegend=True,
|
| 96 |
+
)
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
fig.update_layout(
|
| 100 |
+
autosize=False,
|
| 101 |
+
width=650,
|
| 102 |
+
height=600,
|
| 103 |
+
title=f"Average Score Vs Throughput (A100-80GB, Float16, Batch Size <b>{bs}</b>)",
|
| 104 |
+
xaxis_title=f"{throughput_column}",
|
| 105 |
+
yaxis_title="Average Code Score",
|
| 106 |
+
)
|
| 107 |
+
return fig
|