File size: 9,832 Bytes
ceec7c0 07ff622 4d1e2f9 ceec7c0 07ff622 ceec7c0 4d71b60 cd9bf75 4d71b60 ceec7c0 4d71b60 ceec7c0 4d71b60 56c60c2 a907c4f 4d71b60 07ff622 ceec7c0 0114908 ceec7c0 07ff622 ceec7c0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 |
import json
import pickle
from datetime import datetime, date
import gradio as gr
import pandas as pd
import plotly.graph_objects as go
def create_big_five_capex_plot():
# Capex in Millions of USD per Quarter of Microsoft, Google, Meta, Apple, Amazon
big_five_capex = []
with open("big_five_capex.jsonl", 'r') as file:
for line in file:
big_five_capex.append(json.loads(line))
df = pd.DataFrame(big_five_capex)
fig = go.Figure()
companies = ['Microsoft', 'Google', 'Meta', 'Apple', 'Amazon']
colors = ['#80bb00', '#ee161f', '#0065e3', '#000000', '#ff6200']
for company, color in zip(companies, colors):
fig.add_trace(go.Bar(
x=df['Quarter'],
y=df[company],
name=company,
marker_color=color
))
fig.update_layout(
title='Capital Expenditure of the Big Five Tech Companies in Millions of U.S. Dollars per Quarter',
xaxis_title='Quarter',
yaxis_title='Capex (Millions of U.S. Dollars)',
barmode='stack',
legend_title='Companies',
height=800
)
return fig
def create_chip_designers_data_center_revenue_plot():
# Data Center Revenue in Millions of USD per Quarter of NVIDIA, AMD and Intel
data_center_revenue_by_company = []
with open("chip_designers_data_center_revenue.jsonl", 'r') as file:
for line in file:
data_center_revenue_by_company.append(json.loads(line))
df = pd.DataFrame(data_center_revenue_by_company)
fig = go.Figure()
companies = ['NVIDIA', 'AMD', 'Intel']
colors = ['#80bb00', '#ee161f', '#0065e3'] # TODO
for company, color in zip(companies, colors):
fig.add_trace(go.Bar(
x=df['Quarter'],
y=df[company],
name=company,
marker_color=color
))
fig.update_layout(
title='Data Center Revenue of NVIDIA, AMD and Intel in Millions of U.S. Dollars per Quarter',
xaxis_title='Quarter',
yaxis_title='Data Center Revenue (Millions of U.S. Dollars)',
barmode='stack',
legend_title='Companies',
height=800
)
return fig
def create_size_for_performance_plot(category_to_display: str,
parameter_type_to_display: str,
model_to_compare: str) -> (go.Figure, gr.Dropdown, gr.Dropdown):
with open('elo_results_20240823.pkl', 'rb') as file:
elo_results = pickle.load(file)
categories: list[str] = list(elo_results["text"].keys())
if category_to_display not in categories:
raise gr.Error(message=f"Category '{category_to_display}' not found.")
elo_ratings_for_category: dict = dict(elo_results["text"][category_to_display]["elo_rating_final"])
models: list[dict] = []
with open("models.jsonl", 'r') as file:
for line in file:
models.append(json.loads(line))
size_for_performance_data: list[dict] = []
for model_name, model_elo_rating in elo_ratings_for_category.items():
model_entries_found = [model for model in models if model["Name"] == model_name]
if model_entries_found:
size_for_performance_data.append({
"Name": model_name,
"Release Date": model_entries_found[0]["Release Date"],
"ELO Rating": model_elo_rating,
parameter_type_to_display: model_entries_found[0][parameter_type_to_display]
})
else:
print(f"[WARNING] Model '{model_name}' not found in models.jsonl")
comparison_model_elo_score = elo_ratings_for_category[model_to_compare]
filtered_models = [model for model in size_for_performance_data
if model[parameter_type_to_display] > 0 and
model['ELO Rating'] >= comparison_model_elo_score]
filtered_models.sort(key=lambda x: datetime.strptime(x['Release Date'], "%Y-%m-%d"))
x_dates = [datetime.strptime(model['Release Date'], "%Y-%m-%d") for model in filtered_models]
y_params = []
min_param = float('inf')
for model in filtered_models:
param = model[parameter_type_to_display]
if param <= min_param:
min_param = param
y_params.append(min_param)
fig = go.Figure()
fig.add_trace(go.Scatter(
x=x_dates,
y=y_params,
mode='lines',
line=dict(shape='hv', width=2),
name='Model Parameters'
))
fig.update_layout(
title=f'Model Size Progression for Open-Weights Models Reaching Performance of "{model_to_compare}" in "{category_to_display}" Category',
xaxis_title='Release Date',
yaxis_title=parameter_type_to_display,
yaxis_type='log',
hovermode='x unified',
xaxis=dict(
range=[date(2023, 2, 27), date(2024, 8, 23)],
type='date'
),
height=800
)
for i, model in enumerate(filtered_models):
if i == 0 or y_params[i] < y_params[i - 1]:
fig.add_trace(go.Scatter(
x=[x_dates[i]],
y=[y_params[i]],
mode='markers+text',
marker=dict(size=10),
text=[model['Name']],
textposition="top center",
name=model['Name']
))
return (fig,
gr.Dropdown(choices=categories, value=category_to_display, interactive=True),
gr.Dropdown(choices=list(elo_ratings_for_category.keys()), value=model_to_compare, interactive=True))
with gr.Blocks() as demo:
with gr.Tab("Finance"):
with gr.Tab("Big Five Capex"):
big_five_capex_plot: gr.Plot = gr.Plot()
big_five_capex_button: gr.Button = gr.Button("Show")
with gr.Tab("Chip Designers Data Center Revenue"):
chip_designers_data_center_revenue_plot: gr.Plot = gr.Plot()
chip_designers_data_center_revenue_button: gr.Button = gr.Button("Show")
with gr.Tab("Model Efficiency"):
with gr.Tab("Parameters Necessary for Specific Performance Level"):
with gr.Row():
size_for_performance_category_dropdown: gr.Dropdown = gr.Dropdown(label="Category",
value="full",
choices=["full"],
interactive=False)
size_for_performance_parameter_number_dropdown: gr.Dropdown = gr.Dropdown(label="Parameter Number",
choices=["Total Parameters",
"Active Parameters"],
value="Total Parameters",
interactive=True)
size_for_performance_comparison_model_dropdown: gr.Dropdown = gr.Dropdown(label="Model for Comparison",
value="gpt-4-0314",
choices=["gpt-4-0314"],
interactive=False)
size_for_performance_plot: gr.Plot = gr.Plot()
size_for_performance_button: gr.Button = gr.Button("Show")
size_for_performance_markdown: gr.Markdown = gr.Markdown(
value="""Model performance as reported on [LMSYS Chatbot Arena Leaderboard](https://lmarena.ai/?leaderboard)."""
)
with gr.Tab("API Cost for Specific Performance Level", interactive=False):
api_cost_for_performance_plot: gr.Plot = gr.Plot()
api_cost_for_performance_button: gr.Button = gr.Button("Show")
with gr.Tab("AI System Performance", interactive=False):
with gr.Tab("SWE-bench"):
swe_bench_plot: gr.Plot = gr.Plot()
swe_bench_button: gr.Button = gr.Button("Show")
with gr.Tab("GAIA"):
gaia_plot: gr.Plot = gr.Plot()
gaia_button: gr.Button = gr.Button("Show")
with gr.Tab("Frontier Language Model Training Runs", interactive=False):
with gr.Tab("Street Price of GPUs Used"):
gpu_street_price_plot: gr.Plot = gr.Plot()
gpu_street_price_button: gr.Button = gr.Button("Show")
with gr.Tab("TDP of GPUs Used"):
tdp_gpus_plot: gr.Plot = gr.Plot()
tdp_gpus_button: gr.Button = gr.Button("Show")
big_five_capex_button.click(fn=create_big_five_capex_plot, outputs=big_five_capex_plot)
chip_designers_data_center_revenue_button.click(fn=create_chip_designers_data_center_revenue_plot,
outputs=chip_designers_data_center_revenue_plot)
size_for_performance_button.click(fn=create_size_for_performance_plot,
inputs=[size_for_performance_category_dropdown,
size_for_performance_parameter_number_dropdown,
size_for_performance_comparison_model_dropdown],
outputs=[size_for_performance_plot,
size_for_performance_category_dropdown,
size_for_performance_comparison_model_dropdown])
if __name__ == "__main__":
demo.launch()
|