ranarag commited on
Commit
e693106
·
verified ·
1 Parent(s): dd6c8af

Update README.md

Browse files

Added drop scores

Files changed (1) hide show
  1. README.md +13 -11
README.md CHANGED
@@ -191,7 +191,7 @@ By redesigning a common household item like the plastic bottle, we can create a
191
  <th style="text-align:center; background-color: #001d6c; color: white;">PopQA</th>
192
  <th style="text-align:center; background-color: #001d6c; color: white;">TruthfulQA</th>
193
  <!-- <th style="text-align:center; background-color: #001d6c; color: white;">BigBenchHard</th> -->
194
- <!-- <th style="text-align:center; background-color: #001d6c; color: white;">DROP</th> -->
195
  <th style="text-align:center; background-color: #001d6c; color: white;">GSM8K</th>
196
  <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval</th>
197
  <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval+</th>
@@ -207,7 +207,7 @@ By redesigning a common household item like the plastic bottle, we can create a
207
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.55</td>
208
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.79</td>
209
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">54.46</td> -->
210
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">18.68</td> -->
211
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.55</td>
212
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.45</td>
213
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">75.26</td>
@@ -222,7 +222,7 @@ By redesigning a common household item like the plastic bottle, we can create a
222
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.56</td>
223
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.8</td>
224
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.27</td> -->
225
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">21.12</td> -->
226
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.02</td>
227
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.13</td>
228
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">73.39</td>
@@ -237,7 +237,7 @@ By redesigning a common household item like the plastic bottle, we can create a
237
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 18.4 </td>
238
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 58.97 </td>
239
  <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 52.51 </td> -->
240
- <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 35.98 </td> -->
241
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 72.48 </td>
242
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.51 </td>
243
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 75.68 </td>
@@ -253,7 +253,7 @@ By redesigning a common household item like the plastic bottle, we can create a
253
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.79</td>
254
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.79</td>
255
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.66</td> -->
256
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">61.48</td> -->
257
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">83.24</td>
258
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.32</td>
259
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.15</td>
@@ -269,7 +269,7 @@ By redesigning a common household item like the plastic bottle, we can create a
269
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">13.25</td>
270
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.43</td>
271
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.71</td> -->
272
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">44.46</td> -->
273
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.18</td>
274
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.54</td>
275
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">62.91</td>
@@ -285,7 +285,7 @@ By redesigning a common household item like the plastic bottle, we can create a
285
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">18.12</td>
286
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">63.06</td>
287
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">70.40</td> -->
288
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">54.71</td> -->
289
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">84.46</td>
290
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">93.35</td>
291
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.91</td>
@@ -301,7 +301,7 @@ By redesigning a common household item like the plastic bottle, we can create a
301
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">9.94</td>
302
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.14</td>
303
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.04</td> -->
304
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">42.76</td> -->
305
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.47</td>
306
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.89</td>
307
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.43</td>
@@ -316,7 +316,7 @@ By redesigning a common household item like the plastic bottle, we can create a
316
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.7</td>
317
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.84</td>
318
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">68.55</td> -->
319
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">50.78</td> -->
320
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.15</td>
321
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.63</td>
322
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.79</td>
@@ -332,7 +332,7 @@ By redesigning a common household item like the plastic bottle, we can create a
332
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.04</td>
333
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">66.92</td>
334
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.77</td> -->
335
- <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">50.95</td> -->
336
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">81.65</td>
337
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.35</td>
338
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.72</td>
@@ -347,7 +347,7 @@ By redesigning a common household item like the plastic bottle, we can create a
347
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 26.17 </td>
348
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 66.86 </td>
349
  <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.01 </td> -->
350
- <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 41.53 </td> -->
351
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.89 </td>
352
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 89.73 </td>
353
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 86.09 </td>
@@ -414,6 +414,8 @@ Granite-3.3-8B-Instruct builds upon Granite-3.3-8B-Base, leveraging both permiss
414
  - 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
415
 
416
  <p><a href="#fnref1" title="Jump back to reference">[1]</a> Evaluated using <a href="https://github.com/allenai/olmes">OLMES</a> (except AttaQ and Arena-Hard scores)</p>
 
 
417
  <!-- ## Citation
418
  ```
419
  @misc{granite-models,
 
191
  <th style="text-align:center; background-color: #001d6c; color: white;">PopQA</th>
192
  <th style="text-align:center; background-color: #001d6c; color: white;">TruthfulQA</th>
193
  <!-- <th style="text-align:center; background-color: #001d6c; color: white;">BigBenchHard</th> -->
194
+ <th style="text-align:center; background-color: #001d6c; color: white;">DROP<sup id="fnref2"><a href="#fn2">2</a></sup></th>
195
  <th style="text-align:center; background-color: #001d6c; color: white;">GSM8K</th>
196
  <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval</th>
197
  <th style="text-align:center; background-color: #001d6c; color: white;">HumanEval+</th>
 
207
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.55</td>
208
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.79</td>
209
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">54.46</td> -->
210
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.99</td>
211
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.55</td>
212
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.45</td>
213
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">75.26</td>
 
222
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">20.56</td>
223
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">59.8</td>
224
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.27</td> -->
225
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">23.84</td>
226
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.02</td>
227
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.13</td>
228
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">73.39</td>
 
237
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 18.4 </td>
238
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 58.97 </td>
239
  <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 52.51 </td> -->
240
+ <td style="text-align:center; background-color: #DAE8FF; color: black;"> 44.33 </td>
241
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 72.48 </td>
242
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.51 </td>
243
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 75.68 </td>
 
253
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.79</td>
254
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">52.79</td>
255
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.66</td> -->
256
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">71.23</td>
257
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">83.24</td>
258
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.32</td>
259
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">80.15</td>
 
269
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">13.25</td>
270
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.43</td>
271
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.71</td> -->
272
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">49.73</td>
273
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">72.18</td>
274
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">67.54</td>
275
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">62.91</td>
 
285
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">18.12</td>
286
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">63.06</td>
287
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">70.40</td> -->
288
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.06</td>
289
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">84.46</td>
290
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">93.35</td>
291
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.91</td>
 
301
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">9.94</td>
302
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">47.14</td>
303
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.04</td> -->
304
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">51.78</td>
305
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.47</td>
306
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.89</td>
307
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">78.43</td>
 
316
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.7</td>
317
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">65.84</td>
318
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">68.55</td> -->
319
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">58.57</td>
320
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">79.15</td>
321
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.63</td>
322
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.79</td>
 
332
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">28.04</td>
333
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">66.92</td>
334
  <!-- <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">64.77</td> -->
335
+ <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">58.29</td>
336
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">81.65</td>
337
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">89.35</td>
338
  <td style="text-align:center; background-color: #FFFFFF; color: #2D2D2D;">85.72</td>
 
347
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 26.17 </td>
348
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 66.86 </td>
349
  <!-- <td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.01 </td> -->
350
+ <td style="text-align:center; background-color: #DAE8FF; color: black;"> 59.36 </td>
351
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 80.89 </td>
352
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 89.73 </td>
353
  <td style="text-align:center; background-color: #DAE8FF; color: black;"> 86.09 </td>
 
414
  - 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
415
 
416
  <p><a href="#fnref1" title="Jump back to reference">[1]</a> Evaluated using <a href="https://github.com/allenai/olmes">OLMES</a> (except AttaQ and Arena-Hard scores)</p>
417
+ <p><a href="#fnref2" title="Jump back to reference">[2]</a> Modified the implementation to handle some of the issues mentioned <a href="https://huggingface.co/blog/open-llm-leaderboard-drop">here</a></p>
418
+ <!-- ## Citation
419
  <!-- ## Citation
420
  ```
421
  @misc{granite-models,