added gpt 4.5 and flash lite

data_loader.py

@@ -604,9 +604,9 @@ HEADER_CONTENT = (
 
 CARDS = """        <div class="metrics-grid">
             <div class="metric-card">
-                <div class="metric-number metric-blue">18</div>
+                <div class="metric-number metric-blue">20</div>
                 <div class="metric-label">Total Models</div>
-                <div class="metric-detail primary">13 Private</div>
+                <div class="metric-detail primary">15 Private</div>
                 <div class="metric-detail primary">5 Open Source</div>
             </div>
 
@@ -1003,11 +1003,11 @@ METHODOLOGY = """
             <tbody>
                 <tr>
                     <td>Performance Champion</td>
-                    <td>Claude 3.7 Sonnet comes at the top with 0.953 but Gemini-2.0-flash dominates with 0.938 score at a very affordable cost, excelling in both complex tasks and safety features.</td>
+                    <td>Claude 3.7 Sonnet(0.953) comes at the top but Gemini-2.0-flash(0.938) & Gemini-2.0-flash-lite(0.933) dominate at a very affordable cost, excelling in both complex tasks and safety features.</td>
                 </tr>
                 <tr>
                     <td>Price-Performance Paradox</td>
-                    <td>Top 3 models span 10x price difference yet only 4% performance gap, challenging pricing assumptions</td>
+                    <td>Top 3 models span 10x price difference yet only 2% performance gap, challenging pricing assumptions</td>
                 </tr>
                 <tr>
                     <td>Open Vs Closed Source</td>
@@ -1278,7 +1278,7 @@ evaluate_handler.finish()
         </div>
         <h3 class="feature-title">Updated Periodically</h3>
         <ul class="feature-list">
-            <li>12 private models evaluated</li>
+            <li>15 private models evaluated</li>
             <li>5 open source models included</li>
             <li>Monthly model additions</li>
         </ul>

get_exp_data.ipynb

@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -38,8 +38,6 @@
     "    rows = pq.get_rows(\n",
     "        project_id=PROJECT_ID,\n",
     "        run_id=run_id,\n",
-    "        task_type=None,\n",
-    "        config=None,\n",
     "        starting_token=0,\n",
     "        limit=1000,\n",
     "    )\n",
@@ -127,7 +125,7 @@
     "        ))\n",
     "\n",
     "\n",
-    "models = [\"accounts/fireworks/models/qwen2p5-72b-instruct\", \"meta-llama/Llama-3.3-70B-Instruct-Turbo\", \"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\"]\n",
+    "models = [\"gpt-4.5-preview-2025-02-27\", \"gemini-2.0-flash-lite-001\"]\n",
     "# models = load_data()[\"Model\"]\n",
     "\n",
     "# Process each model sequentially, but datasets in parallel\n",

results.csv

@@ -1,7 +1,9 @@
 Model,Model Type,Model Output Type,Vendor,Input cost per million token,Output cost per million token,Model Avg,single turn perf,multi turn perf,BFCL_v3_multi_turn_base_multi_func_call,BFCL_v3_multi_turn_composite,tau_long_context,xlam_single_tool_multiple_call,BFCL_v3_multi_turn_miss_param,xlam_multiple_tool_single_call,xlam_tool_miss,BFCL_v3_multi_turn_long_context,BFCL_v3_irrelevance,BFCL_v3_multi_turn_base_single_func_call,xlam_single_tool_single_call,xlam_multiple_tool_multiple_call,BFCL_v3_multi_turn_miss_func,toolace_single_func_call
 claude-3-7-sonnet-20250219,Private,Reasoning,Anthropic,3,15,0.953,0.96,0.95,0.92,0.96,1,0.95,0.97,1,0.96,0.94,0.97,0.96,0.99,0.82,0.92,0.975
 gemini-2.0-flash-001,Private,Normal,Google,0.15,0.6,0.938,0.95,0.93,0.91,0.94,0.9,0.96,0.92,0.95,0.89,0.91,0.98,0.93,0.97,0.98,0.93,0.965
+gemini-2.0-flash-lite-001,Private,Normal,Google,0.075,0.3,0.933,0.96,0.91,0.81,0.98,0.98,0.9,0.91,0.92,0.98,0.86,0.99,0.87,0.97,0.96,0.95,0.975
 gpt-4o-2024-11-20,Private,Normal,OpenAI,2.5,10,0.900,0.92,0.88,0.85,0.9,0.92,0.95,0.88,0.99,0.63,0.83,0.98,0.89,0.98,0.98,0.86,0.965
+gpt-4.5-preview-2025-02-27,Private,Normal,OpenAI,75,150,0.900,0.93,0.87,0.85,0.91,0.92,0.97,0.92,0.99,0.67,0.85,0.98,0.85,1,0.98,0.8,0.915
 gemini-1.5-flash,Private,Normal,Google,0.075,0.3,0.895,0.88,0.91,0.9,0.9,0.89,0.87,0.91,0.83,0.71,0.87,0.98,0.89,0.94,0.93,0.92,0.99
 gemini-1.5-pro,Private,Normal,Google,1.25,5,0.885,0.87,0.91,0.89,0.93,0.75,0.97,0.9,0.87,0.57,0.91,0.94,0.92,0.99,0.97,0.86,0.925
 o1-2024-12-17,Private,Reasoning,OpenAI,15,60,0.876,0.83,0.92,0.89,0.92,0.98,0.71,0.91,0.99,0.73,0.88,0.98,0.96,1,0.43,0.94,0.95
@@ -17,4 +19,4 @@ mistral-small-2409,Private,Normal,Mistral,0.2,0.6,0.750,0.82,0.68,0.7,0.77,0.72,
 ministral-8b-2410,Private,Normal,Mistral,0.1,0.1,0.689,0.73,0.65,0.75,0.59,0.73,0.98,0.66,0.98,0.34,0.78,0.24,0.81,0.9,0.95,0.53,0.41
 Meta-Llama-3.1-8B-Instruct-Turbo,Open source,Normal,Meta,0.2,0.2,0.678,0.71,0.64,0.77,0.49,0.44,0.96,0.66,0.98,0.25,0.73,0.48,0.76,0.93,0.96,0.51,0.575
 open-mistral-nemo-2407,Open source,Normal,Mistral,0.15,0.15,0.661,0.68,0.64,0.7,0.64,0.51,0.98,0.68,0.99,0.26,0.78,0.21,0.75,0.9,0.94,0.51,0.41
-Dataset Avg,,,,,,,0.83,0.80,0.81,0.79,0.78,0.89,0.81,0.96,0.62,0.81,0.82,0.82,0.92,0.85,0.74,0.81
+Dataset Avg,,,,,,,0.84,0.81,0.82,0.81,0.79,0.89,0.82,0.96,0.64,0.82,0.84,0.83,0.93,0.86,0.76,0.82