import json import pickle from datetime import datetime, date import gradio as gr import pandas as pd import plotly.graph_objects as go def create_big_five_capex_plot(): # Capex in Millions of USD per Quarter of Microsoft, Google, Meta, Apple, Amazon big_five_capex = [] with open("big_five_capex.jsonl", 'r') as file: for line in file: big_five_capex.append(json.loads(line)) df = pd.DataFrame(big_five_capex) fig = go.Figure() companies = ['Microsoft', 'Google', 'Meta', 'Apple', 'Amazon'] colors = ['#80bb00', '#ee161f', '#0065e3', '#000000', '#ff6200'] for company, color in zip(companies, colors): fig.add_trace(go.Bar( x=df['Quarter'], y=df[company], name=company, marker_color=color )) fig.update_layout( title='Capital Expenditure of the Big Five Tech Companies in Millions of U.S. Dollars per Quarter', xaxis_title='Quarter', yaxis_title='Capex (Millions of U.S. Dollars)', barmode='stack', legend_title='Companies', height=800 ) return fig def create_chip_designers_data_center_revenue_plot(): # Data Center Revenue in Millions of USD per Quarter of NVIDIA, AMD and Intel data_center_revenue_by_company = [] with open("chip_designers_data_center_revenue.jsonl", 'r') as file: for line in file: data_center_revenue_by_company.append(json.loads(line)) df = pd.DataFrame(data_center_revenue_by_company) fig = go.Figure() companies = ['NVIDIA', 'AMD', 'Intel'] colors = ['#80bb00', '#ee161f', '#0065e3'] # TODO for company, color in zip(companies, colors): fig.add_trace(go.Bar( x=df['Quarter'], y=df[company], name=company, marker_color=color )) fig.update_layout( title='Data Center Revenue of NVIDIA, AMD and Intel in Millions of U.S. Dollars per Quarter', xaxis_title='Quarter', yaxis_title='Data Center Revenue (Millions of U.S. Dollars)', barmode='stack', legend_title='Companies', height=800 ) return fig def create_size_for_performance_plot(category_to_display: str, parameter_type_to_display: str, model_to_compare: str) -> (go.Figure, gr.Dropdown, gr.Dropdown): with open('elo_results_20240823.pkl', 'rb') as file: elo_results = pickle.load(file) categories: list[str] = list(elo_results["text"].keys()) if category_to_display not in categories: raise gr.Error(message=f"Category '{category_to_display}' not found.") elo_ratings_for_category: dict = dict(elo_results["text"][category_to_display]["elo_rating_final"]) models: list[dict] = [] with open("models.jsonl", 'r') as file: for line in file: models.append(json.loads(line)) size_for_performance_data: list[dict] = [] for model_name, model_elo_rating in elo_ratings_for_category.items(): model_entries_found = [model for model in models if model["Name"] == model_name] if model_entries_found: size_for_performance_data.append({ "Name": model_name, "Release Date": model_entries_found[0]["Release Date"], "ELO Rating": model_elo_rating, parameter_type_to_display: model_entries_found[0][parameter_type_to_display] }) else: print(f"[WARNING] Model '{model_name}' not found in models.jsonl") comparison_model_elo_score = elo_ratings_for_category[model_to_compare] filtered_models = [model for model in size_for_performance_data if model[parameter_type_to_display] > 0 and model['ELO Rating'] >= comparison_model_elo_score] filtered_models.sort(key=lambda x: datetime.strptime(x['Release Date'], "%Y-%m-%d")) x_dates = [datetime.strptime(model['Release Date'], "%Y-%m-%d") for model in filtered_models] y_params = [] min_param = float('inf') for model in filtered_models: param = model[parameter_type_to_display] if param <= min_param: min_param = param y_params.append(min_param) fig = go.Figure() fig.add_trace(go.Scatter( x=x_dates, y=y_params, mode='lines', line=dict(shape='hv', width=2), name='Model Parameters' )) fig.update_layout( title=f'Model Size Progression for Open-Weights Models Reaching Performance of "{model_to_compare}" in "{category_to_display}" Category', xaxis_title='Release Date', yaxis_title=parameter_type_to_display, yaxis_type='log', hovermode='x unified', xaxis=dict( range=[date(2023, 2, 27), date(2024, 8, 23)], type='date' ), height=800 ) for i, model in enumerate(filtered_models): if i == 0 or y_params[i] < y_params[i - 1]: fig.add_trace(go.Scatter( x=[x_dates[i]], y=[y_params[i]], mode='markers+text', marker=dict(size=10), text=[model['Name']], textposition="top center", name=model['Name'] )) return (fig, gr.Dropdown(choices=categories, value=category_to_display, interactive=True), gr.Dropdown(choices=list(elo_ratings_for_category.keys()), value=model_to_compare, interactive=True)) with gr.Blocks() as demo: with gr.Tab("Finance"): with gr.Tab("Big Five Capex"): big_five_capex_plot: gr.Plot = gr.Plot() big_five_capex_button: gr.Button = gr.Button("Show") with gr.Tab("Chip Designers Data Center Revenue"): chip_designers_data_center_revenue_plot: gr.Plot = gr.Plot() chip_designers_data_center_revenue_button: gr.Button = gr.Button("Show") with gr.Tab("Model Efficiency"): with gr.Tab("Parameters Necessary for Specific Performance Level"): with gr.Row(): size_for_performance_category_dropdown: gr.Dropdown = gr.Dropdown(label="Category", value="full", choices=["full"], interactive=False) size_for_performance_parameter_number_dropdown: gr.Dropdown = gr.Dropdown(label="Parameter Number", choices=["Total Parameters", "Active Parameters"], value="Total Parameters", interactive=True) size_for_performance_comparison_model_dropdown: gr.Dropdown = gr.Dropdown(label="Model for Comparison", value="gpt-4-0314", choices=["gpt-4-0314"], interactive=False) size_for_performance_plot: gr.Plot = gr.Plot() size_for_performance_button: gr.Button = gr.Button("Show") size_for_performance_markdown: gr.Markdown = gr.Markdown( value="""Model performance as reported on [LMSYS Chatbot Arena Leaderboard](https://lmarena.ai/?leaderboard).""" ) with gr.Tab("API Cost for Specific Performance Level", interactive=False): api_cost_for_performance_plot: gr.Plot = gr.Plot() api_cost_for_performance_button: gr.Button = gr.Button("Show") with gr.Tab("AI System Performance", interactive=False): with gr.Tab("SWE-bench"): swe_bench_plot: gr.Plot = gr.Plot() swe_bench_button: gr.Button = gr.Button("Show") with gr.Tab("GAIA"): gaia_plot: gr.Plot = gr.Plot() gaia_button: gr.Button = gr.Button("Show") with gr.Tab("Frontier Language Model Training Runs", interactive=False): with gr.Tab("Street Price of GPUs Used"): gpu_street_price_plot: gr.Plot = gr.Plot() gpu_street_price_button: gr.Button = gr.Button("Show") with gr.Tab("TDP of GPUs Used"): tdp_gpus_plot: gr.Plot = gr.Plot() tdp_gpus_button: gr.Button = gr.Button("Show") big_five_capex_button.click(fn=create_big_five_capex_plot, outputs=big_five_capex_plot) chip_designers_data_center_revenue_button.click(fn=create_chip_designers_data_center_revenue_plot, outputs=chip_designers_data_center_revenue_plot) size_for_performance_button.click(fn=create_size_for_performance_plot, inputs=[size_for_performance_category_dropdown, size_for_performance_parameter_number_dropdown, size_for_performance_comparison_model_dropdown], outputs=[size_for_performance_plot, size_for_performance_category_dropdown, size_for_performance_comparison_model_dropdown]) if __name__ == "__main__": demo.launch()