[ { "model_name": "model-BioMistral-BioMistral-7B", "2008 Fall": 23.232323232323232, "2012 Spring": 21.1340206185567, "2010 Fall": 23.115577889447238, "2009 Spring": 25.757575757575758, "2010 Spring": 23.0, "2011 Spring": 23.737373737373737, "2012 Fall": 28.57142857142857, "2011 Fall": 24.242424242424242, "2009 Fall": 23.5, "2022 Spring": 20.2020202020202, "2014 Spring": 24.489795918367346, "2022 Fall": 28.57142857142857, "2014 Fall": 22.279792746113987, "2024 Spring": 25.888324873096447, "2015 Spring": 24.731182795698924, "2023 Spring": 24.747474747474747, "2021 Fall": 26.767676767676768, "2015 Fall": 28.421052631578945, "2023 Fall": 25.64102564102564, "2021 Spring": 19.387755102040817, "2013 Spring": 23.711340206185564, "2013 Fall": 29.84293193717277, "overall_accuracy": 24.576467857971686 }, { "model_name": "model-johnsnowlabs-JSL-MedLlama-3-8B-v2.0", "2008 Fall": 29.292929292929294, "2012 Spring": 31.958762886597935, "2010 Fall": 28.14070351758794, "2009 Spring": 31.313131313131315, "2010 Spring": 35.0, "2011 Spring": 23.737373737373737, "2012 Fall": 38.265306122448976, "2011 Fall": 25.757575757575758, "2009 Fall": 35.0, "2022 Spring": 30.808080808080806, "2014 Spring": 31.122448979591837, "2022 Fall": 34.69387755102041, "2014 Fall": 34.196891191709845, "2024 Spring": 29.949238578680205, "2015 Spring": 31.182795698924732, "2023 Spring": 30.808080808080806, "2021 Fall": 29.797979797979796, "2015 Fall": 26.31578947368421, "2023 Fall": 32.82051282051282, "2021 Spring": 33.6734693877551, "2013 Spring": 30.927835051546392, "2013 Fall": 30.89005235602094, "overall_accuracy": 31.16732420515201 }, { "model_name": "model-meta-llama-Meta-Llama-3.1-8B-Instruct", "2008 Fall": 40.4040404040404, "2012 Spring": 40.72164948453608, "2010 Fall": 43.71859296482412, "2009 Spring": 47.97979797979798, "2010 Spring": 46.0, "2011 Spring": 33.83838383838384, "2012 Fall": 50.0, "2011 Fall": 41.41414141414141, "2009 Fall": 54.0, "2022 Spring": 38.88888888888889, "2014 Spring": 33.16326530612245, "2022 Fall": 40.816326530612244, "2014 Fall": 42.487046632124354, "2024 Spring": 39.59390862944163, "2015 Spring": 42.473118279569896, "2023 Spring": 44.94949494949495, "2021 Fall": 40.909090909090914, "2015 Fall": 41.578947368421055, "2023 Fall": 42.05128205128205, "2021 Spring": 41.3265306122449, "2013 Spring": 41.23711340206185, "2013 Fall": 44.50261780104712, "overall_accuracy": 42.37642144349037 }, { "model_name": "model-speakleash-Bielik-11B-v2.2-Instruct", "2008 Fall": 45.45454545454545, "2012 Spring": 46.391752577319586, "2010 Fall": 52.26130653266332, "2009 Spring": 51.515151515151516, "2010 Spring": 48.5, "2011 Spring": 41.91919191919192, "2012 Fall": 57.6530612244898, "2011 Fall": 45.95959595959596, "2009 Fall": 49.5, "2022 Spring": 40.4040404040404, "2014 Spring": 34.69387755102041, "2022 Fall": 43.36734693877551, "2014 Fall": 44.04145077720207, "2024 Spring": 50.76142131979695, "2015 Spring": 43.54838709677419, "2023 Spring": 41.91919191919192, "2021 Fall": 43.93939393939394, "2015 Fall": 40.0, "2023 Fall": 37.94871794871795, "2021 Spring": 50.0, "2013 Spring": 47.93814432989691, "2013 Fall": 42.93193717277487, "overall_accuracy": 45.50939893246693 }, { "model_name": "model-gpt-4o-2024-08-06", "2008 Fall": 78.78787878787878, "2012 Spring": 77.83505154639175, "2010 Fall": 74.37185929648241, "2009 Spring": 85.85858585858585, "2010 Spring": 83.0, "2011 Spring": 72.22222222222221, "2012 Fall": 76.53061224489795, "2011 Fall": 72.72727272727273, "2009 Fall": 81.5, "2022 Spring": 77.27272727272727, "2014 Spring": 79.08163265306123, "2022 Fall": 69.38775510204081, "2014 Fall": 77.2020725388601, "2024 Spring": 70.55837563451777, "2015 Spring": 72.04301075268818, "2023 Spring": 72.72727272727273, "2021 Fall": 72.22222222222221, "2015 Fall": 75.78947368421053, "2023 Fall": 69.74358974358974, "2021 Spring": 73.9795918367347, "2013 Spring": 76.28865979381443, "2013 Fall": 74.3455497382199, "overall_accuracy": 75.63239730796009 }, { "model_name": "model-OpenMeditron-Meditron3-8B", "2008 Fall": 38.88888888888889, "2012 Spring": 39.69072164948454, "2010 Fall": 36.18090452261307, "2009 Spring": 41.41414141414141, "2010 Spring": 43.0, "2011 Spring": 32.82828282828283, "2012 Fall": 42.3469387755102, "2011 Fall": 33.33333333333333, "2009 Fall": 41.5, "2022 Spring": 34.34343434343434, "2014 Spring": 30.102040816326532, "2022 Fall": 36.734693877551024, "2014 Fall": 39.89637305699482, "2024 Spring": 38.07106598984771, "2015 Spring": 36.55913978494624, "2023 Spring": 38.38383838383838, "2021 Fall": 41.41414141414141, "2015 Fall": 39.473684210526315, "2023 Fall": 34.35897435897436, "2021 Spring": 40.816326530612244, "2013 Spring": 42.78350515463917, "2013 Fall": 40.83769633507853, "overall_accuracy": 38.31515432815038 }, { "model_name": "model-Qwen-Qwen2.5-7B-Instruct", "2008 Fall": 41.41414141414141, "2012 Spring": 40.20618556701031, "2010 Fall": 43.21608040201005, "2009 Spring": 40.4040404040404, "2010 Spring": 41.5, "2011 Spring": 38.38383838383838, "2012 Fall": 48.97959183673469, "2011 Fall": 38.88888888888889, "2009 Fall": 43.5, "2022 Spring": 43.93939393939394, "2014 Spring": 35.714285714285715, "2022 Fall": 42.857142857142854, "2014 Fall": 46.63212435233161, "2024 Spring": 45.68527918781726, "2015 Spring": 42.473118279569896, "2023 Spring": 44.94949494949495, "2021 Fall": 43.43434343434344, "2015 Fall": 40.0, "2023 Fall": 46.15384615384615, "2021 Spring": 47.44897959183674, "2013 Spring": 44.84536082474227, "2013 Fall": 43.97905759162304, "overall_accuracy": 42.93339521930842 }, { "model_name": "model-mistralai-Mistral-Large-Instruct-2407", "2008 Fall": 61.61616161616161, "2012 Spring": 54.123711340206185, "2010 Fall": 62.311557788944725, "2009 Spring": 67.67676767676768, "2010 Spring": 69.5, "2011 Spring": 56.56565656565656, "2012 Fall": 66.3265306122449, "2011 Fall": 58.080808080808076, "2009 Fall": 63.0, "2022 Spring": 54.54545454545454, "2014 Spring": 54.08163265306123, "2022 Fall": 56.63265306122449, "2014 Fall": 56.476683937823836, "2024 Spring": 55.83756345177665, "2015 Spring": 58.602150537634415, "2023 Spring": 57.57575757575758, "2021 Fall": 56.060606060606055, "2015 Fall": 60.0, "2023 Fall": 48.205128205128204, "2021 Spring": 57.14285714285714, "2013 Spring": 61.34020618556701, "2013 Fall": 55.497382198952884, "overall_accuracy": 58.71431886748666 }, { "model_name": "model-gpt-4o-mini-2024-07-18", "2008 Fall": 59.09090909090909, "2012 Spring": 55.670103092783506, "2010 Fall": 55.27638190954774, "2009 Spring": 68.18181818181817, "2010 Spring": 62.5, "2011 Spring": 50.0, "2012 Fall": 65.3061224489796, "2011 Fall": 55.05050505050505, "2009 Fall": 64.0, "2022 Spring": 52.52525252525253, "2014 Spring": 46.93877551020408, "2022 Fall": 54.08163265306123, "2014 Fall": 57.51295336787565, "2024 Spring": 56.34517766497462, "2015 Spring": 52.68817204301075, "2023 Spring": 53.535353535353536, "2021 Fall": 58.080808080808076, "2015 Fall": 57.89473684210527, "2023 Fall": 58.46153846153847, "2021 Spring": 59.693877551020414, "2013 Spring": 55.154639175257735, "2013 Fall": 51.30890052356021, "overall_accuracy": 56.811325133441635 }, { "model_name": "model-meta-llama-Llama-3.2-1B-Instruct", "2008 Fall": 24.242424242424242, "2012 Spring": 20.618556701030926, "2010 Fall": 23.618090452261306, "2009 Spring": 24.242424242424242, "2010 Spring": 23.0, "2011 Spring": 24.747474747474747, "2012 Fall": 28.57142857142857, "2011 Fall": 21.71717171717172, "2009 Fall": 25.5, "2022 Spring": 21.21212121212121, "2014 Spring": 22.448979591836736, "2022 Fall": 29.591836734693878, "2014 Fall": 23.316062176165804, "2024 Spring": 33.50253807106599, "2015 Spring": 23.655913978494624, "2023 Spring": 28.28282828282828, "2021 Fall": 27.27272727272727, "2015 Fall": 24.736842105263158, "2023 Fall": 27.692307692307693, "2021 Spring": 22.448979591836736, "2013 Spring": 23.711340206185564, "2013 Fall": 27.748691099476442, "overall_accuracy": 25.08702715247157 }, { "model_name": "model-OpenMeditron-Meditron3-70B", "2008 Fall": 45.95959595959596, "2012 Spring": 43.29896907216495, "2010 Fall": 46.733668341708544, "2009 Spring": 51.010101010101, "2010 Spring": 49.0, "2011 Spring": 42.42424242424242, "2012 Fall": 47.44897959183674, "2011 Fall": 46.96969696969697, "2009 Fall": 49.0, "2022 Spring": 45.95959595959596, "2014 Spring": 41.83673469387755, "2022 Fall": 48.46938775510204, "2014 Fall": 47.15025906735752, "2024 Spring": 52.28426395939086, "2015 Spring": 50.53763440860215, "2023 Spring": 47.474747474747474, "2021 Fall": 48.98989898989899, "2015 Fall": 51.05263157894737, "2023 Fall": 42.56410256410256, "2021 Spring": 45.40816326530612, "2013 Spring": 49.48453608247423, "2013 Fall": 45.54973821989529, "overall_accuracy": 47.203527500580186 }, { "model_name": "model-aaditya-Llama3-OpenBioLLM-70B", "2008 Fall": 19.19191919191919, "2012 Spring": 19.072164948453608, "2010 Fall": 35.175879396984925, "2009 Spring": 50.0, "2010 Spring": 43.0, "2011 Spring": 38.88888888888889, "2012 Fall": 43.36734693877551, "2011 Fall": 35.35353535353536, "2009 Fall": 48.0, "2022 Spring": 45.95959595959596, "2014 Spring": 39.285714285714285, "2022 Fall": 43.36734693877551, "2014 Fall": 39.37823834196891, "2024 Spring": 44.16243654822335, "2015 Spring": 41.39784946236559, "2023 Spring": 42.92929292929293, "2021 Fall": 42.92929292929293, "2015 Fall": 38.421052631578945, "2023 Fall": 38.97435897435898, "2021 Spring": 41.83673469387755, "2013 Spring": 41.23711340206185, "2013 Fall": 42.93193717277487, "overall_accuracy": 39.777210489672775 }, { "model_name": "model-meta-llama-Meta-Llama-3.1-70B-Instruct", "2008 Fall": 62.121212121212125, "2012 Spring": 61.34020618556701, "2010 Fall": 62.8140703517588, "2009 Spring": 69.1919191919192, "2010 Spring": 73.0, "2011 Spring": 59.59595959595959, "2012 Fall": 69.89795918367348, "2011 Fall": 65.65656565656566, "2009 Fall": 68.0, "2022 Spring": 68.68686868686868, "2014 Spring": 54.59183673469388, "2022 Fall": 62.755102040816325, "2014 Fall": 60.62176165803109, "2024 Spring": 61.42131979695431, "2015 Spring": 60.215053763440864, "2023 Spring": 62.62626262626263, "2021 Fall": 64.14141414141415, "2015 Fall": 58.42105263157895, "2023 Fall": 61.02564102564103, "2021 Spring": 68.36734693877551, "2013 Spring": 62.371134020618555, "2013 Fall": 57.06806282722513, "overall_accuracy": 63.402181480621955 }, { "model_name": "model-meta-llama-Llama-3.2-3B-Instruct", "2008 Fall": 36.36363636363637, "2012 Spring": 30.927835051546392, "2010 Fall": 36.18090452261307, "2009 Spring": 34.34343434343434, "2010 Spring": 40.5, "2011 Spring": 32.82828282828283, "2012 Fall": 40.816326530612244, "2011 Fall": 29.292929292929294, "2009 Fall": 43.0, "2022 Spring": 29.797979797979796, "2014 Spring": 28.061224489795915, "2022 Fall": 36.224489795918366, "2014 Fall": 29.533678756476682, "2024 Spring": 35.53299492385787, "2015 Spring": 36.55913978494624, "2023 Spring": 35.35353535353536, "2021 Fall": 27.77777777777778, "2015 Fall": 30.526315789473685, "2023 Fall": 29.230769230769234, "2021 Spring": 31.122448979591837, "2013 Spring": 33.50515463917525, "2013 Fall": 35.07853403141361, "overall_accuracy": 33.766535158969596 }, { "model_name": "model-ProbeMedicalYonseiMAILab-medllama3-v20", "2008 Fall": 30.303030303030305, "2012 Spring": 31.958762886597935, "2010 Fall": 30.15075376884422, "2009 Spring": 37.37373737373738, "2010 Spring": 31.5, "2011 Spring": 30.808080808080806, "2012 Fall": 34.69387755102041, "2011 Fall": 30.808080808080806, "2009 Fall": 35.5, "2022 Spring": 34.84848484848485, "2014 Spring": 26.53061224489796, "2022 Fall": 34.183673469387756, "2014 Fall": 35.233160621761655, "2024 Spring": 35.025380710659896, "2015 Spring": 33.33333333333333, "2023 Spring": 35.35353535353536, "2021 Fall": 31.818181818181817, "2015 Fall": 37.89473684210527, "2023 Fall": 31.28205128205128, "2021 Spring": 42.857142857142854, "2013 Spring": 38.659793814432994, "2013 Fall": 39.26701570680628, "overall_accuracy": 34.04502204687863 }, { "model_name": "model-mistralai-Mistral-Small-Instruct-2409", "2008 Fall": 38.38383838383838, "2012 Spring": 34.5360824742268, "2010 Fall": 40.7035175879397, "2009 Spring": 44.44444444444444, "2010 Spring": 46.5, "2011 Spring": 34.84848484848485, "2012 Fall": 46.42857142857143, "2011 Fall": 38.88888888888889, "2009 Fall": 45.0, "2022 Spring": 38.88888888888889, "2014 Spring": 34.69387755102041, "2022 Fall": 42.857142857142854, "2014 Fall": 37.82383419689119, "2024 Spring": 44.16243654822335, "2015 Spring": 38.17204301075269, "2023 Spring": 41.41414141414141, "2021 Fall": 42.42424242424242, "2015 Fall": 42.10526315789473, "2023 Fall": 40.0, "2021 Spring": 41.3265306122449, "2013 Spring": 43.29896907216495, "2013 Fall": 44.50261780104712, "overall_accuracy": 40.98398700394523 }, { "model_name": "model-Qwen-Qwen2.5-72B-Instruct", "2008 Fall": 58.58585858585859, "2012 Spring": 56.70103092783505, "2010 Fall": 60.80402010050251, "2009 Spring": 60.1010101010101, "2010 Spring": 63.5, "2011 Spring": 58.58585858585859, "2012 Fall": 66.83673469387756, "2011 Fall": 54.54545454545454, "2009 Fall": 70.5, "2022 Spring": 59.59595959595959, "2014 Spring": 53.57142857142857, "2022 Fall": 61.73469387755102, "2014 Fall": 61.6580310880829, "2024 Spring": 57.360406091370564, "2015 Spring": 58.602150537634415, "2023 Spring": 56.56565656565656, "2021 Fall": 59.59595959595959, "2015 Fall": 57.36842105263158, "2023 Fall": 54.871794871794876, "2021 Spring": 58.673469387755105, "2013 Spring": 61.855670103092784, "2013 Fall": 57.06806282722513, "overall_accuracy": 59.503365049895564 } ]