add scores for 1B

README.md

@@ -94,131 +94,150 @@ We report in the following table our internal pipeline benchmarks:
         <col style="width: 10%;">
         <col style="width: 7%;">
         <col style="width: 7%;">
+        <col style="width: 7%;">
         <col style="background-color: rgba(80, 15, 213, 0.5); width: 7%;">
     </colgroup>
     <thead>
         <tr>
             <th>Category</th>
             <th>Benchmark</th>
-            <th>Llama-3.1-8B-Instruct</th>
-            <th>Qwen2.5-7B-Instruct</th>
-            <th>Falcon3-7B-Instruct</th>
+            <th>Llama-3.2-1B</th>
+            <th>Qwen2.5-1.5B</th>
+            <th>SmolLM2-1.7B</th>
+            <th>Falcon3-1B-Instruct</th>
         </tr>
     </thead>
     <tbody>
         <tr>
             <td rowspan="3">General</td>
             <td>MMLU (5-shot)</td>
-            <td>55.9</td>
-            <td><b>72.4</b></td>
-            <td>68</td>
+            <td>23.4</td>
+            <td><b>58.4</b></td>
+            <td>48.4</td>
+            <td>43.9</td>
         </tr>
         <tr>
             <td>MMLU-PRO (5-shot)</td>
-            <td>21.8</td>
-            <td>35.8</td>
-            <td><b>40.7</b></td>
+            <td>11.3</td>
+            <td><b>21.3</b></td>
+            <td>17.2</td>
+            <td>18.6</td>
         </tr>
         <tr>
             <td>IFEval</td>
-            <td><b>78.8</b></td>
-            <td>74.7</td>
-            <td>76.5</td>
+            <td><b>55.8</b></td>
+            <td>44.4</td>
+            <td>53.0</td>
+            <td>54.4</td>
         </tr>
         <tr>
             <td rowspan="3">Math</td>
             <td>GSM8K (5-shot)</td>
-            <td>78.1</td>
-            <td>77.5</td>
-            <td><b>79.1</b></td>
+            <td>37.4</td>
+            <td><b>57.2</b></td>
+            <td>43.4</td>
+            <td>38.6</td>
         </tr>
         <tr>
             <td>GSM8K (8-shot, COT)</td>
-            <td>79.8</td>
-            <td>72.7</td>
-            <td><b>80.9</b></td>
+            <td>35.6</td>
+            <td><b>62.2</b></td>
+            <td>47.2</td>
+            <td>41.8</td>
         </tr>
         <tr>
             <td>MATH Lvl-5 (4-shot)</td>
-            <td>10.4</td>
-            <td>26</td>
-            <td><b>33.1</b></td>
+            <td><b>3.9</b></td>
+            <td>0.2</td>
+            <td>0.1</td>
+            <td>1.0</td>
         </tr>
         <tr>
-            <td rowspan="5">Reasoning</td>
+            <td rowspan="6">Reasoning</td>
             <td>Arc Challenge (25-shot)</td>
-            <td>46.6</td>
-            <td>55.7</td>
-            <td><b>65.9</b></td>
+            <td>34.1</td>
+            <td>47.0</td>
+            <td><b>47.6</b></td>
+            <td>45.9</td>
         </tr>
         <tr>
             <td>GPQA (0-shot)</td>
-            <td><b>33.6</b></td>
-            <td>31.9</td>
-            <td>32</td>
+            <td>25.3</td>
+            <td><b>29.6</b></td>
+            <td>28.7</td>
+            <td>26.5</td>
         </tr>
         <tr>
             <td>GPQA (0-shot, COT)</td>
-            <td>9.6</td>
-            <td>13.8</td>
-            <td><b>22.3</b></td>
+            <td>13.2</td>
+            <td>9.2</td>
+            <td>16.0</td>
+            <td><b>21.3</b></td>
         </tr>
         <tr>
             <td>MUSR (0-shot)</td>
-            <td>38.6</td>
-            <td>40.7</td>
-            <td><b>46.4</b></td>
+            <td>32.4</td>
+            <td>36.8</td>
+            <td>33.0</td>
+            <td><b>40.7</b></td>
         </tr>
         <tr>
             <td>BBH (3-shot)</td>
-            <td>43.7</td>
-            <td><b>53.9</b></td>
-            <td>52.4</td>
+            <td>30.3</td>
+            <td><b>38.5</b></td>
+            <td>33.1</td>
+            <td>35.1</td>
         </tr>
         <tr>
-            <td rowspan="4">CommonSense Understanding</td>
+            <td>BBH (3-shot, COT)</td>
+            <td>0.0</td>
+            <td>20.3</td>
+            <td>0.8</td>
+            <td><b>30.5</b></td>
+        </tr>
+        <tr>
+            <td rowspan="5">CommonSense Understanding</td>
             <td>PIQA (0-shot)</td>
-            <td><b>78.9</b></td>
-            <td>73.7</td>
-            <td>78.8</td>
+            <td>72.1</td>
+            <td>73.2</td>
+            <td><b>74.4</b></td>
+            <td>72.0</td>
         </tr>
         <tr>
             <td>SciQ (0-shot)</td>
-            <td>80.2</td>
-            <td>50.9</td>
-            <td><b>94.7</b></td>
+            <td>61.8</td>
+            <td>69.5</td>
+            <td>71.4</td>
+            <td><b>86.8</b></td>
         </tr>
         <tr>
             <td>Winogrande (0-shot)</td>
             <td>-</td>
             <td>-</td>
-            <td>70.4</td>
+            <td>-</td>
+            <td><b>60.2</b></td>
         </tr>
         <tr>
             <td>OpenbookQA (0-shot)</td>
-            <td><b>46.2</b></td>
-            <td>42.4</td>
-            <td>45.8</td>
+            <td>40.2</td>
+            <td>40.4</td>
+            <td><b>42.8</b></td>
+            <td>40.0</td>
         </tr>
         <tr>
-            <td rowspan="2">Instructions following</td>
             <td>MT-Bench (avg)</td>
-            <td>7.86</td>
-            <td><b>8.54</b></td>
-            <td>8.36</td>
+            <td>5.4</td>
+            <td><b>7.1</b></td>
+            <td>6.1</td>
+            <td>5.5</td>
         </tr>
         <tr>
+            <td rowspan="1">Instructions following</td>
             <td>Alapaca (WC)</td>
-            <td>26.57</td>
-            <td><b>31.5</b></td>
-            <td>26.13</td>
-        </tr>
-        <tr>
-            <td>Tool use</td>
-            <td>BFCL AST (avg)</td>
-            <td>90.6</td>
-            <td><b>91.4</b></td>
-            <td>72.3</td>
+            <td><b>8.6</b></td>
+            <td><b>8.6</b></td>
+            <td>5.4</td>
+            <td>6.1</td>
         </tr>
     </tbody>
 </table>