updated layout

app.py

@@ -4,10 +4,10 @@ import gradio as gr
 
 from src.control_panel import create_control_panel, create_control_callback
 from src.latency_score_memory import create_lat_score_mem_plot
+from src.quantization_kernels import create_quant_plots
 from src.leaderboard import create_leaderboard_table
 from src.bettertransformer import create_bt_plots
 from src.flashattentionv2 import create_fa2_plots
-from src.custom_kernels import create_custom_kernels_plots
 from src.llm_perf import get_llm_perf_df
 from src.assets import custom_css
 from src.content import (
@@ -52,18 +52,14 @@ with demo:
                     ####################### LEADERBOARD TAB #######################
                     with gr.TabItem("Leaderboard 🏅", id=0):
                         leaderboard_table = create_leaderboard_table(llm_perf_df)
-                    ####################### LAT. vs. SCORE vs. MEM. TAB #######################
-                    with gr.TabItem("Latency vs. Score vs. Memory 📊", id=1):
                         lat_score_mem_plot = create_lat_score_mem_plot(llm_perf_df)
                     ####################### BETTERTRANSFORMER SPEEDUP TAB #######################
-                    with gr.TabItem("BetterTransformer Speedup 📈", id=2):
+                    with gr.TabItem("BetterTransformer 📈", id=2):
                         bt_prefill_plot, bt_decode_plot = create_bt_plots(llm_perf_df)
-                    with gr.TabItem("FlashAttentionV2 Speedup 📈", id=3):
+                    with gr.TabItem("FlashAttentionV2 📈", id=3):
                         fa2_prefill_plot, fa2_decode_plot = create_fa2_plots(llm_perf_df)
-                    with gr.TabItem("Custom Quantization Kernels Comparison 🏆", id=4):
-                        custom_kernels_prefill_plot, custom_kernels_decode_plot = create_custom_kernels_plots(
-                            llm_perf_df
-                        )
+                    with gr.TabItem("Custom Quantization Kernels 📈", id=4):
+                        quant_prefill_plot, quant_decode_plot = create_quant_plots(llm_perf_df)
 
                 ####################### CONTROL CALLBACK #######################
                 create_control_callback(
@@ -84,8 +80,8 @@ with demo:
                     bt_decode_plot,
                     fa2_prefill_plot,
                     fa2_decode_plot,
-                    custom_kernels_prefill_plot,
-                    custom_kernels_decode_plot,
+                    quant_prefill_plot,
+                    quant_decode_plot,
                 )
         ####################### ABOUT TAB #######################
         with gr.TabItem("About 📖", id=3):

src/bettertransformer.py

@@ -14,7 +14,9 @@ BETTERTRANSFORMER_DATA = [
     # deployment settings
     "DType 📥",
     "Backend 🏭",
+    "Optimization 🛠️",
     "Quantization 🗜️",
+    "Optimization 🛠️ BetterTransformer",
     # primary measurements
     "Prefill Latency (s)",
     "Prefill Latency (s) BetterTransformer",
@@ -29,10 +31,10 @@ BETTERTRANSFORMER_DATA = [
 
 
 def get_bt_df(llm_perf_df):
-    bt_df = llm_perf_df.copy()
+    copy_df = llm_perf_df.copy()
     # seperate original model experiments from BetterTransformer experiments
-    original_df = bt_df[(bt_df["Optimization 🛠️"] == "None") & (bt_df["DType 📥"] == "float16")]
-    bt_df = bt_df[bt_df["Optimization 🛠️"] == "BetterTransformer"]
+    original_df = copy_df[(copy_df["Optimization 🛠️"] == "None") & (copy_df["DType 📥"] == "float16")]
+    bt_df = copy_df[(copy_df["Optimization 🛠️"] == "BetterTransformer") & (copy_df["DType 📥"] == "float16")]
     # merge the two dataframes
     bt_df = pd.merge(
         original_df,
@@ -54,78 +56,78 @@ def get_bt_df(llm_perf_df):
     return bt_df
 
 
-def get_bt_decode_fig(llm_perf_df):
+def get_bt_prefill_fig(llm_perf_df):
     bt_df = get_bt_df(llm_perf_df)
     # plot
-    decode_fig = px.box(
+    prefill_fig = px.box(
         bt_df,
         x="Arch 🏛️",
-        y="Decode Throughput Speedup (%)",
+        y="Prefill Latency Speedup (%)",
         color_discrete_sequence=px.colors.qualitative.Light24,
         custom_data=BETTERTRANSFORMER_DATA,
         color="Quantization 🗜️",
         points="all",
     )
     # add hover data
-    decode_fig.update_traces(
+    prefill_fig.update_traces(
         hovertemplate="<br>".join(
             [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(BETTERTRANSFORMER_DATA)]
         )
     )
     # add layout
-    decode_fig.update_layout(
+    prefill_fig.update_layout(
         title={
-            "text": "Decode Throughput Speedup per Architecture",
+            "text": "Prefill Latency Speedup per Architecture, Compared To Non-Optimized Model",
             "y": 0.95,
             "x": 0.5,
             "xanchor": "center",
             "yanchor": "top",
         },
         xaxis_title="LLM Architecture",
-        yaxis_title="Decode Speedup (%)",
+        yaxis_title="Prefill Speedup (%)",
         legend_title="Quantization Scheme",
         width=1200,
         height=600,
     )
 
-    return decode_fig
+    return prefill_fig
 
 
-def get_bt_prefill_fig(llm_perf_df):
+def get_bt_decode_fig(llm_perf_df):
     bt_df = get_bt_df(llm_perf_df)
     # plot
-    prefill_fig = px.box(
+    decode_fig = px.box(
         bt_df,
         x="Arch 🏛️",
-        y="Prefill Latency Speedup (%)",
+        y="Decode Throughput Speedup (%)",
         color_discrete_sequence=px.colors.qualitative.Light24,
         custom_data=BETTERTRANSFORMER_DATA,
         color="Quantization 🗜️",
         points="all",
     )
     # add hover data
-    prefill_fig.update_traces(
+    decode_fig.update_traces(
         hovertemplate="<br>".join(
             [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(BETTERTRANSFORMER_DATA)]
         )
     )
     # add layout
-    prefill_fig.update_layout(
+    decode_fig.update_layout(
         title={
-            "text": "Prefill Latency Speedup per Architecture",
+            "text": "Decode Throughput Speedup per Architecture, Compared To Non-Optimized Model",
             "y": 0.95,
             "x": 0.5,
             "xanchor": "center",
             "yanchor": "top",
         },
         xaxis_title="LLM Architecture",
-        yaxis_title="Prefill Speedup (%)",
+        yaxis_title="Decode Speedup (%)",
         legend_title="Quantization Scheme",
         width=1200,
         height=600,
     )
 
-    return prefill_fig
+    return decode_fig
 
 
 def create_bt_plots(llm_perf_df):

src/control_panel.py

@@ -5,7 +5,7 @@ from src.leaderboard import get_leaderboard_df
 from src.latency_score_memory import get_lat_score_mem_fig
 from src.bettertransformer import get_bt_prefill_fig, get_bt_decode_fig
 from src.flashattentionv2 import get_fa2_prefill_fig, get_fa2_decode_fig
-from src.custom_kernels import get_custom_kernels_prefill_fig, get_custom_kernels_decode_fig
+from src.quantization_kernels import get_quant_prefill_fig, get_quant_decode_fig
 
 
 def create_control_panel(machine: str = "hf-dgx-01"):
@@ -133,8 +133,8 @@ def filter_fn(
     filtered_bt_decode_fig = get_bt_decode_fig(filtered_df)
     filtered_fa2_prefill_fig = get_fa2_prefill_fig(filtered_df)
     filtered_fa2_decode_fig = get_fa2_decode_fig(filtered_df)
-    filtered_custom_kernels_prefill_fig = get_custom_kernels_prefill_fig(filtered_df)
-    filtered_custom_kernels_decode_fig = get_custom_kernels_decode_fig(filtered_df)
+    filtered_quant_prefill_fig = get_quant_prefill_fig(filtered_df)
+    filtered_quant_decode_fig = get_quant_decode_fig(filtered_df)
 
     return [
         filtered_leaderboard_df,
@@ -143,8 +143,8 @@ def filter_fn(
         filtered_bt_decode_fig,
         filtered_fa2_prefill_fig,
         filtered_fa2_decode_fig,
-        filtered_custom_kernels_prefill_fig,
-        filtered_custom_kernels_decode_fig,
+        filtered_quant_prefill_fig,
+        filtered_quant_decode_fig,
     ]
 
 
@@ -167,8 +167,8 @@ def create_control_callback(
     bt_decode_plot,
     fa2_prefill_plot,
     fa2_decode_plot,
-    exllama_prefill_plot,
-    exllama_decode_plot,
+    quant_prefill_plot,
+    quant_decode_plot,
 ):
     filter_button.click(
         fn=filter_fn,
@@ -189,7 +189,7 @@ def create_control_callback(
             bt_decode_plot,
             fa2_prefill_plot,
             fa2_decode_plot,
-            exllama_prefill_plot,
-            exllama_decode_plot,
+            quant_prefill_plot,
+            quant_decode_plot,
         ],
     )

src/flashattentionv2.py

@@ -14,7 +14,9 @@ FLASHATTENTIONV2_DATA = [
     # deployment settings
     "DType 📥",
     "Backend 🏭",
+    "Optimization 🛠️",
     "Quantization 🗜️",
+    "Optimization 🛠️ FlashAttentionV2",
     # primary measurements
     "Prefill Latency (s)",
     "Prefill Latency (s) FlashAttentionV2",
@@ -29,10 +31,10 @@ FLASHATTENTIONV2_DATA = [
 
 
 def get_fa2_df(llm_perf_df):
-    fa2_df = llm_perf_df.copy()
+    copy_df = llm_perf_df.copy()
     # seperate original model experiments from FlashAttentionV2 experiments
-    original_df = fa2_df[(fa2_df["Optimization 🛠️"] == "None") & (fa2_df["DType 📥"] == "float16")]
-    fa2_df = fa2_df[fa2_df["Optimization 🛠️"] == "FlashAttentionV2"]
+    original_df = copy_df[(copy_df["Optimization 🛠️"] == "None") & (copy_df["DType 📥"] == "float16")]
+    fa2_df = copy_df[(copy_df["Optimization 🛠️"] == "FlashAttentionV2") & (copy_df["DType 📥"] == "float16")]
     # merge the two dataframes
     fa2_df = pd.merge(
         original_df,
@@ -47,7 +49,6 @@ def get_fa2_df(llm_perf_df):
     fa2_df["Decode Throughput Speedup (%)"] = (
         (fa2_df["Decode Throughput (tokens/s) FlashAttentionV2"] / fa2_df["Decode Throughput (tokens/s)"]) * 100
     ).round(2) - 100
-
     # filter speedups > 1000%
     fa2_df = fa2_df[fa2_df["Prefill Latency Speedup (%)"] < 1000]
     fa2_df = fa2_df[fa2_df["Decode Throughput Speedup (%)"] < 1000]
@@ -76,7 +77,7 @@ def get_fa2_decode_fig(llm_perf_df):
     # add layout
     decode_fig.update_layout(
         title={
-            "text": "Decode Throughput Speedup per Architecture",
+            "text": "Decode Throughput Speedup per Architecture, Compared To Non-Optimized Model",
             "y": 0.95,
             "x": 0.5,
             "xanchor": "center",
@@ -113,7 +114,7 @@ def get_fa2_prefill_fig(llm_perf_df):
     # add layout
     prefill_fig.update_layout(
         title={
-            "text": "Prefill Latency Speedup per Architecture",
+            "text": "Prefill Latency Speedup per Architecture, Compared To Non-Optimized Model",
             "y": 0.95,
             "x": 0.5,
             "xanchor": "center",

src/latency_score_memory.py

@@ -8,6 +8,8 @@ SCORE_MEMORY_LATENCY_DATA = [
     "Params (B)",
     "DType 📥",
     "Backend 🏭",
+    "Optimization 🛠️",
+    "Quantization 🗜️",
     "Open LLM Score (%)",
     "Prefill Latency (s)",
     "Decode Throughput (tokens/s)",
@@ -42,7 +44,7 @@ def get_lat_score_mem_fig(llm_perf_df):
             "xanchor": "center",
             "yanchor": "top",
         },
-        xaxis_title="Per 256 Tokens Latency (s)",
+        xaxis_title="Time To Generate 256 Tokens (s)",
         yaxis_title="Open LLM Score (%)",
         legend_title="LLM Architecture",
         width=1200,

src/{custom_kernels.py → quantization_kernels.py}

@@ -3,7 +3,7 @@ import pandas as pd
 import plotly.express as px
 
 
-CUSTOM_KERNELS_DATA = [
+QUANT_DATA = [
     # open llm
     "Model 🤗",
     "Arch 🏛️",
@@ -29,13 +29,13 @@ CUSTOM_KERNELS_DATA = [
 ]
 
 
-def get_custom_kernels_df(llm_perf_df):
+def get_quant_df(llm_perf_df):
     copy_df = llm_perf_df.copy()
     # seperate vanilla GPTQ experiments from Custom Kernel experiments
     vanilla_df = copy_df[
-        (copy_df["Backend 🏭"] == "pytorch") & 
+        (copy_df["Backend 🏭"] == "pytorch") &
         (copy_df["Quantization 🗜️"] == "None") &
-        (copy_df["Optimization 🛠️"] == "None") & 
+        (copy_df["Optimization 🛠️"] == "None") &
         (copy_df["DType 📥"] == "float16")
     ]
     exllamav1_df = copy_df[(copy_df["Quantization 🗜️"] == "GPTQ.4bit+ExllamaV1")]
@@ -68,42 +68,36 @@ def get_custom_kernels_df(llm_perf_df):
         suffixes=["", " Custom Kernel"],
     )
     # concat the two dataframes row-wise
-    custom_kernels_df = pd.concat([exllamav1_df, exllamav2_df, gemm_df, gemv_df])
+    quant_df = pd.concat([exllamav1_df, exllamav2_df, gemm_df, gemv_df])
     # compute speedups
-    custom_kernels_df["Prefill Latency Speedup (%)"] = (
-        (custom_kernels_df["Prefill Latency (s)"] / custom_kernels_df["Prefill Latency (s) Custom Kernel"]) * 100
+    quant_df["Prefill Latency Speedup (%)"] = (
+        (quant_df["Prefill Latency (s)"] / quant_df["Prefill Latency (s) Custom Kernel"]) * 100
     ).round(2) - 100
-    custom_kernels_df["Decode Throughput Speedup (%)"] = (
-        (
-            custom_kernels_df["Decode Throughput (tokens/s) Custom Kernel"]
-            / custom_kernels_df["Decode Throughput (tokens/s)"]
-        )
-        * 100
+    quant_df["Decode Throughput Speedup (%)"] = (
+        (quant_df["Decode Throughput (tokens/s) Custom Kernel"] / quant_df["Decode Throughput (tokens/s)"]) * 100
     ).round(2) - 100
     # filter speedups > 1000%
-    custom_kernels_df = custom_kernels_df[custom_kernels_df["Prefill Latency Speedup (%)"] < 1000]
-    custom_kernels_df = custom_kernels_df[custom_kernels_df["Decode Throughput Speedup (%)"] < 1000]
+    quant_df = quant_df[quant_df["Prefill Latency Speedup (%)"] < 1000]
+    quant_df = quant_df[quant_df["Decode Throughput Speedup (%)"] < 1000]
 
-    return custom_kernels_df
+    return quant_df
 
 
-def get_custom_kernels_decode_fig(llm_perf_df):
-    custom_kernels_df = get_custom_kernels_df(llm_perf_df)
+def get_quant_decode_fig(llm_perf_df):
+    quant_df = get_quant_df(llm_perf_df)
     # plot
     decode_fig = px.box(
-        custom_kernels_df,
+        quant_df,
         x="Arch 🏛️",
         y="Decode Throughput Speedup (%)",
         color_discrete_sequence=px.colors.qualitative.Light24,
-        custom_data=CUSTOM_KERNELS_DATA,
+        custom_data=QUANT_DATA,
         color="Quantization 🗜️ Custom Kernel",
         points="all",
     )
     # add hover data
     decode_fig.update_traces(
-        hovertemplate="<br>".join(
-            [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(CUSTOM_KERNELS_DATA)]
-        )
+        hovertemplate="<br>".join([f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(QUANT_DATA)])
     )
     # add layout
     decode_fig.update_layout(
@@ -124,23 +118,21 @@ def get_custom_kernels_decode_fig(llm_perf_df):
     return decode_fig
 
 
-def get_custom_kernels_prefill_fig(llm_perf_df):
-    custom_kernels_df = get_custom_kernels_df(llm_perf_df)
+def get_quant_prefill_fig(llm_perf_df):
+    quant_df = get_quant_df(llm_perf_df)
     # plot
     prefill_fig = px.box(
-        custom_kernels_df,
+        quant_df,
         x="Arch 🏛️",
         y="Prefill Latency Speedup (%)",
         color_discrete_sequence=px.colors.qualitative.Light24,
-        custom_data=CUSTOM_KERNELS_DATA,
+        custom_data=QUANT_DATA,
         color="Quantization 🗜️ Custom Kernel",
         points="all",
     )
     # add hover data
     prefill_fig.update_traces(
-        hovertemplate="<br>".join(
-            [f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(CUSTOM_KERNELS_DATA)]
-        )
+        hovertemplate="<br>".join([f"<b>{column}:</b> %{{customdata[{i}]}}" for i, column in enumerate(QUANT_DATA)])
     )
     # add layout
     prefill_fig.update_layout(
@@ -161,12 +153,12 @@ def get_custom_kernels_prefill_fig(llm_perf_df):
     return prefill_fig
 
 
-def create_custom_kernels_plots(llm_perf_df):
+def create_quant_plots(llm_perf_df):
     # descriptive text
     gr.HTML("👆 Hover over the points 👆 for additional information.", elem_id="text")
     # get figures
-    prefill_fig = get_custom_kernels_prefill_fig(llm_perf_df)
-    decode_fig = get_custom_kernels_decode_fig(llm_perf_df)
+    prefill_fig = get_quant_prefill_fig(llm_perf_df)
+    decode_fig = get_quant_decode_fig(llm_perf_df)
 
     # create plots
     prefill_plot = gr.components.Plot(value=prefill_fig, elem_id="plot", show_label=False)