init leaderboard

.gitignore

@@ -0,0 +1,2 @@
+*ipynb
+__pycache__

README.md

@@ -1,12 +1,33 @@
 ---
 title: Openvlm Subjective Leaderboard
-emoji: 🐠
-colorFrom: purple
+emoji: 🌎
+colorFrom: blue
 colorTo: green
 sdk: gradio
-sdk_version: 5.28.0
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
+tags:
+- leaderboard
+short_description: 'VLMEvalKit Evaluation Subjectivce Benchmark Results Collection '
+sdk_version: 4.44.1
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+In this leaderboard, we display subjective benchmark evaluation results obtained with VLMEvalKit. 
+
+Github: https://github.com/open-compass/VLMEvalKit
+Report: https://arxiv.org/abs/2407.11691
+
+Please consider to cite the report if the resource is useful to your research:
+
+```BibTex
+@misc{duan2024vlmevalkitopensourcetoolkitevaluating,
+      title={VLMEvalKit: An Open-Source Toolkit for Evaluating Large Multi-Modality Models}, 
+      author={Haodong Duan and Junming Yang and Yuxuan Qiao and Xinyu Fang and Lin Chen and Yuan Liu and Amit Agarwal and Zhe Chen and Mo Li and Yubo Ma and Hailong Sun and Xiangyu Zhao and Junbo Cui and Xiaoyi Dong and Yuhang Zang and Pan Zhang and Jiaqi Wang and Dahua Lin and Kai Chen},
+      year={2024},
+      eprint={2407.11691},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2407.11691}, 
+}
+```

app.py

@@ -0,0 +1,215 @@
+import abc
+import gradio as gr
+
+from gen_table import *
+from meta_data import *
+
+# import pandas as pd
+# pd.set_option('display.max_colwidth', 0)
+
+head_style = """
+<style>
+@media (min-width: 1536px)
+{
+    .gradio-container {
+        min-width: var(--size-full) !important;
+    }
+}
+</style>
+"""
+
+with gr.Blocks(title="OpenVLM Subjective Leaderboard", head=head_style) as demo:
+    struct = load_results()
+    timestamp = struct['time']
+    EVAL_TIME = format_timestamp(timestamp)
+    results = struct['results']
+    N_MODEL = len(results)
+    N_DATA = len(results['Phi-4-Vision']) - 1
+    DATASETS = list(results['Phi-4-Vision'])
+    DATASETS.remove('META')
+    print(DATASETS)
+
+    gr.Markdown(LEADERBORAD_INTRODUCTION.format(N_MODEL, N_DATA, EVAL_TIME))
+    structs = [abc.abstractproperty() for _ in range(N_DATA)]
+
+    with gr.Tabs(elem_classes='tab-buttons') as tabs:
+        with gr.TabItem('🏅 OpenVLM Subjective Leaderboard', elem_id='main', id=0):
+            gr.Markdown(LEADERBOARD_MD['MAIN'])
+            _, check_box = BUILD_L1_DF(results, MAIN_FIELDS)
+            table = generate_table(results, DEFAULT_BENCH)
+            table['Rank'] = list(range(1, len(table) + 1))
+
+            type_map = check_box['type_map']
+            type_map['Rank'] = 'number'
+
+            checkbox_group = gr.CheckboxGroup(
+                choices=check_box['all'],
+                value=check_box['required'],
+                label='Evaluation Dimension',
+                interactive=True,
+            )
+
+            headers = ['Rank'] + check_box['essential'] + checkbox_group.value
+            with gr.Row():
+                model_name = gr.Textbox(
+                    value='Input the Model Name (fuzzy, case insensitive)', 
+                    label='Model Name', 
+                    interactive=True,
+                    visible=True)
+                model_size = gr.CheckboxGroup(
+                    choices=MODEL_SIZE,
+                    value=MODEL_SIZE,
+                    label='Model Size',
+                    interactive=True
+                )
+                model_type = gr.CheckboxGroup(
+                    choices=MODEL_TYPE,
+                    value=MODEL_TYPE,
+                    label='Model Type',
+                    interactive=True
+                )
+            data_component = gr.components.DataFrame(
+                value=table[headers],
+                type='pandas',
+                datatype=[type_map[x] for x in headers],
+                interactive=False,
+                wrap=True,
+                visible=True)
+
+            def filter_df(fields, model_name, model_size, model_type):
+                filter_list = ['Avg Score', 'Avg Rank', 'OpenSource']
+                headers = ['Rank'] + check_box['essential'] + fields
+
+                new_fields = [field for field in fields if field not in filter_list]
+                df = generate_table(results, new_fields)
+
+                df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+                df = df[df['flag']]
+                df.pop('flag')
+                if len(df):
+                    df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+                    df = df[df['flag']]
+                    df.pop('flag')
+                df['Rank'] = list(range(1, len(df) + 1))
+                default_val = 'Input the Model Name (fuzzy, case insensitive)'
+                if model_name != default_val:
+                    print(model_name)
+                    model_name = model_name.lower()
+                    method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Method']]
+                    flag = [model_name in name for name in method_names]
+                    df['TEMP_FLAG'] = flag
+                    df = df[df['TEMP_FLAG'] == True] 
+                    df.pop('TEMP_FLAG')
+
+                comp = gr.components.DataFrame(
+                    value=df[headers],
+                    type='pandas',
+                    datatype=[type_map[x] for x in headers],
+                    interactive=False,
+                    wrap=True, 
+                    visible=True)
+                return comp
+
+            for cbox in [checkbox_group, model_size, model_type]:
+                cbox.change(fn=filter_df, inputs=[checkbox_group, model_name, model_size, model_type], outputs=data_component)
+            model_name.submit(fn=filter_df, inputs=[checkbox_group, model_name, model_size, model_type], outputs=data_component)
+
+        with gr.TabItem('🔍 About', elem_id='about', id=1):
+            gr.Markdown(urlopen(VLMEVALKIT_README).read().decode())
+
+        for i, dataset in enumerate(DATASETS):
+            with gr.TabItem(f'📊 {dataset} Leaderboard', elem_id=dataset, id=i + 2):
+                if dataset in LEADERBOARD_MD:
+                    gr.Markdown(LEADERBOARD_MD[dataset])
+
+                s = structs[i]
+                s.table, s.check_box = BUILD_L2_DF(results, dataset)
+                s.type_map = s.check_box['type_map']
+                s.type_map['Rank'] = 'number'
+
+                s.checkbox_group = gr.CheckboxGroup(
+                    choices=s.check_box['all'],
+                    value=s.check_box['required'],
+                    label=f'{dataset} CheckBoxes',
+                    interactive=True,
+                )
+                s.headers = ['Rank'] + s.check_box['essential'] + s.checkbox_group.value
+                s.table['Rank'] = list(range(1, len(s.table) + 1))
+
+                with gr.Row():
+                    s.model_name = gr.Textbox(
+                        value='Input the Model Name (fuzzy, case insensitive)', 
+                        label='Model Name', 
+                        interactive=True,
+                        visible=True)
+                    s.model_size = gr.CheckboxGroup(
+                        choices=MODEL_SIZE,
+                        value=MODEL_SIZE,
+                        label='Model Size',
+                        interactive=True
+                    )
+                    s.model_type = gr.CheckboxGroup(
+                        choices=MODEL_TYPE,
+                        value=MODEL_TYPE,
+                        label='Model Type',
+                        interactive=True
+                    )
+                s.data_component = gr.components.DataFrame(
+                    value=s.table[s.headers],
+                    type='pandas',
+                    datatype=[s.type_map[x] for x in s.headers],
+                    interactive=False,
+                    wrap=True,
+                    visible=True)
+                s.dataset = gr.Textbox(value=dataset, label=dataset, visible=False)
+
+                def filter_df_l2(dataset_name, fields, model_name, model_size, model_type):
+                    s = structs[DATASETS.index(dataset_name)]
+                    headers = ['Rank'] + s.check_box['essential'] + fields
+                    df = cp.deepcopy(s.table)
+                    df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+                    df = df[df['flag']]
+                    df.pop('flag')
+                    if len(df):
+                        df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+                        df = df[df['flag']]
+                        df.pop('flag')
+                    df['Rank'] = list(range(1, len(df) + 1))
+                    default_val = 'Input the Model Name (fuzzy, case insensitive)'
+                    if model_name != default_val:
+                        print(model_name)
+                        model_name = model_name.lower()
+                        method_names = [x.split('</a>')[0].split('>')[-1].lower() for x in df['Method']]
+                        flag = [model_name in name for name in method_names]
+                        df['TEMP_FLAG'] = flag
+                        df = df[df['TEMP_FLAG'] == True] 
+                        df.pop('TEMP_FLAG')
+
+                    comp = gr.components.DataFrame(
+                        value=df[headers],
+                        type='pandas',
+                        datatype=[s.type_map[x] for x in headers],
+                        interactive=False,
+                        wrap=True,
+                        visible=True)
+                    return comp
+
+                for cbox in [s.checkbox_group, s.model_size, s.model_type]:
+                    cbox.change(
+                        fn=filter_df_l2,
+                        inputs=[s.dataset, s.checkbox_group, s.model_name, s.model_size, s.model_type],
+                        outputs=s.data_component)
+                s.model_name.submit(
+                    fn=filter_df_l2, 
+                    inputs=[s.dataset, s.checkbox_group, s.model_name, s.model_size, s.model_type],
+                    outputs=s.data_component)
+
+    with gr.Row():
+        with gr.Accordion('Citation', open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id='citation-button')
+
+if __name__ == '__main__':
+    demo.launch(server_name='0.0.0.0')

gen_table.py

@@ -0,0 +1,235 @@
+import copy as cp
+import json
+from collections import defaultdict
+from urllib.request import urlopen
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+
+from meta_data import DEFAULT_BENCH, META_FIELDS, URL
+
+
+def listinstr(lst, s):
+    assert isinstance(lst, list)
+    for item in lst:
+        if item in s:
+            return True
+    return False
+
+
+def load_results():
+    data = json.loads(urlopen(URL).read())
+    return data
+
+
+def nth_large(val, vals):
+    return sum([1 for v in vals if v > val]) + 1
+
+
+def format_timestamp(timestamp):
+    date = timestamp[:-6]
+    time = timestamp[-6:]
+    date = date[:-4] + '.' + date[-4:-2] + '.' + date[-2:]
+    time = time[:-4] + ':' + time[-4:-2] + ':' + time[-2:]
+    return date + ' ' + time
+
+
+def model_size_flag(sz, FIELDS):
+    if pd.isna(sz) and 'Unknown' in FIELDS:
+        return True
+    if pd.isna(sz):
+        return False
+    if '<4B' in FIELDS and sz < 4:
+        return True
+    if '4B-10B' in FIELDS and sz >= 4 and sz < 10:
+        return True
+    if '10B-20B' in FIELDS and sz >= 10 and sz < 20:
+        return True
+    if '20B-40B' in FIELDS and sz >= 20 and sz < 40:
+        return True
+    if '>40B' in FIELDS and sz >= 40:
+        return True
+    return False
+
+
+def model_type_flag(line, FIELDS):
+    if 'OpenSource' in FIELDS and line['OpenSource'] == 'Yes':
+        return True
+    if 'API' in FIELDS and line['OpenSource'] == 'No':
+        return True
+    return False
+
+
+def BUILD_L1_DF(results, fields):
+    check_box = {}
+    check_box['essential'] = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'Eval Date']
+    # revise there to set default dataset
+    check_box['required'] = ['Avg Score', 'Avg Rank'] + DEFAULT_BENCH
+    check_box['avg'] = ['Avg Score', 'Avg Rank']
+    check_box['all'] = check_box['avg'] + fields
+    type_map = defaultdict(lambda: 'number')
+    type_map['Method'] = 'html'
+    type_map['Language Model'] = type_map['Vision Model'] = 'html'
+    type_map['OpenSource'] = 'str'
+    type_map['Eval Date'] = 'str'
+    check_box['type_map'] = type_map
+
+    df = generate_table(results, fields)
+    return df, check_box
+
+
+def BUILD_L2_DF(results, dataset):
+    res = defaultdict(list)
+    sub = [v for v in results.values() if dataset in v]
+    assert len(sub)
+    fields = list(sub[0][dataset].keys())
+    if dataset == 'Creation_MMBench':
+        reward_field = [f"{x}:reward" for x in fields]
+        vfs_field = [f"{x}:vfs" for x in fields]
+        fields = reward_field + vfs_field
+        raw_fields = list(sub[0][dataset].keys())
+
+    print(f'fields: {fields}')
+    print(res)  
+
+    non_overall_fields = [x for x in fields if 'Overall' not in x]
+    overall_fields = [x for x in fields if 'Overall' in x]
+    # if dataset == 'MME':
+    #     non_overall_fields = [x for x in non_overall_fields if not listinstr(['Perception', 'Cognition'], x)]
+    #     overall_fields = overall_fields + ['Perception', 'Cognition']
+    # if dataset == 'OCRBench':
+    #     non_overall_fields = [x for x in non_overall_fields if not listinstr(['Final Score'], x)]
+    #     overall_fields = ['Final Score']
+
+    for m in results:
+        item = results[m]
+        if dataset not in item:
+            continue
+        meta = item['META']
+        for k in META_FIELDS:
+            if k == 'Param (B)':
+                param = meta['Parameters']
+                res[k].append(float(param.replace('B', '')) if param != '' else None)
+            elif k == 'Method':
+                name, url = meta['Method']
+                res[k].append(f'<a href="{url}">{name}</a>')
+            elif k == 'Eval Date':
+                eval_date = meta['Time'].split('/')
+                assert len(eval_date) == 3
+                eval_date = [x if len(x) > 1 else '0' + x for x in eval_date]
+                eval_date = '/'.join(eval_date)
+                res[k].append(eval_date)
+            else:
+                res[k].append(meta[k])
+        fields = [x for x in fields]
+
+        if dataset == 'Creation_MMBench':
+            for d in non_overall_fields:
+                original_d, data_type = d.split(':')[0], d.split(':')[-1]
+                res[d].append(item[dataset][original_d][data_type])
+            for d in overall_fields:
+                original_d, data_type = d.split(':')[0], d.split(':')[-1]
+                res[d].append(item[dataset][original_d][data_type])
+                # res[d].append(f"{item[dataset][d]['reward']}/{item[dataset][d]['vfs']}")
+        elif dataset in ['MMAlignBench', 'WildVision']:
+            for d in non_overall_fields:
+                res[d].append(item[dataset][d]['reward'])
+            for d in overall_fields:
+                res[d].append(item[dataset][d]['reward'])
+        else:
+            for d in non_overall_fields:
+                res[d].append(item[dataset][d])
+            for d in overall_fields:
+                res[d].append(item[dataset][d])
+
+    df = pd.DataFrame(res)
+    all_fields = overall_fields + non_overall_fields
+    # Use the first 5 non-overall fields as required fields
+    required_fields = overall_fields if len(overall_fields) else non_overall_fields[:5]
+
+    # if dataset == 'OCRBench':
+    #     df = df.sort_values('Final Score')
+    # elif dataset == 'COCO_VAL':
+    #     df = df.sort_values('CIDEr')
+    # elif dataset == 'VCR':
+    #     df = df.sort_values('Overall-Jaccard')
+    # else:
+    #     df = df.sort_values('Overall')
+    df = df.iloc[::-1]
+
+    check_box = {}
+    check_box['essential'] = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'Eval Date']
+    check_box['required'] = required_fields
+    check_box['all'] = all_fields
+    type_map = defaultdict(lambda: 'number')
+    type_map['Method'] = 'html'
+    type_map['Language Model'] = type_map['Vision Model'] = 'html'
+    type_map['OpenSource'] = 'str'
+    type_map['Eval Date'] = 'str'
+    check_box['type_map'] = type_map
+    return df, check_box
+
+
+def generate_table(results, fields):
+
+    # def get_mmbench_v11(item):
+    #     assert 'MMBench_TEST_CN_V11' in item and 'MMBench_TEST_EN_V11' in item
+    #     val = (item['MMBench_TEST_CN_V11']['Overall'] + item['MMBench_TEST_EN_V11']['Overall']) / 2
+    #     val = float(f'{val:.1f}')
+    #     return val
+
+    res = defaultdict(list)
+    ## item is each model's meta and results
+    for i, m in enumerate(results):
+        item = results[m]
+        meta = item['META']
+        for k in META_FIELDS:
+            if k == 'Param (B)':
+                param = meta['Parameters']
+                res[k].append(float(param.replace('B', '')) if param != '' else None)
+            elif k == 'Method':
+                name, url = meta['Method']
+                res[k].append(f'<a href="{url}">{name}</a>')
+                res['name'].append(name)
+            elif k == 'Eval Date': 
+                eval_date = meta['Time'].split('/')
+                assert len(eval_date) == 3
+                eval_date = [x if len(x) > 1 else '0' + x for x in eval_date]
+                eval_date = '/'.join(eval_date)
+                res[k].append(eval_date)
+            else:
+                res[k].append(meta[k])
+        scores, ranks = [], []
+        # fields is dataset
+        for d in fields:
+            key_name = 'Overall'
+            if d in item:
+                if d in ['Creation_MMBench','MMAlignBench', 'WildVision']:
+                    if d == 'Creation_MMBench':
+                        res[d].append(f"{item[d][key_name]['reward']}/{item[d][key_name]['vfs']}")  # need improve?
+                    else:
+                        res[d].append(item[d][key_name]['reward'])
+                    scores.append((item[d][key_name]['reward']+100)/2)
+                    ranks.append(nth_large(item[d][key_name]['reward'], [x[d][key_name]['reward'] for x in results.values() if d in x]))
+                else:
+                    res[d].append(item[d][key_name])
+                    scores.append(item[d][key_name])
+                    ranks.append(nth_large(item[d][key_name], [x[d][key_name] for x in results.values() if d in x]))
+            else:
+                res[d].append(None)
+                scores.append(None)
+                ranks.append(None)
+
+        res['Avg Score'].append(round(np.mean(scores), 1) if None not in scores else None)
+        res['Avg Rank'].append(round(np.mean(ranks), 2) if None not in ranks else None)
+
+    df = pd.DataFrame(res)
+    valid, missing = df[~pd.isna(df['Avg Score'])], df[pd.isna(df['Avg Score'])]
+    valid = valid.sort_values('Avg Score')
+    valid = valid.iloc[::-1]
+    if len(fields):
+        missing = missing.sort_values(fields[0])
+        missing = missing.iloc[::-1]
+    df = pd.concat([valid, missing])
+    return df

meta_data.py

@@ -0,0 +1,101 @@
+# CONSTANTS-URL
+URL = "http://opencompass.openxlab.space/assets/OpenVLM_Subjective_Leaderboard.json"
+VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
+# CONSTANTS-CITATION
+CITATION_BUTTON_TEXT = r"""@inproceedings{duan2024vlmevalkit,
+  title={Vlmevalkit: An open-source toolkit for evaluating large multi-modality models},
+  author={Duan, Haodong and Yang, Junming and Qiao, Yuxuan and Fang, Xinyu and Chen, Lin and Liu, Yuan and Dong, Xiaoyi and Zang, Yuhang and Zhang, Pan and Wang, Jiaqi and others},
+  booktitle={Proceedings of the 32nd ACM International Conference on Multimedia},
+  pages={11198--11201},
+  year={2024}
+}"""
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+# CONSTANTS-TEXT
+LEADERBORAD_INTRODUCTION = """# OpenVLM Subjective Leaderboard
+### Welcome to the OpenVLM Subjective Leaderboard! On this leaderboard we share the evaluation results of VLMs obtained by the OpenSource Framework:
+### [*VLMEvalKit*: A Toolkit for Evaluating Large Vision-Language Models](https://github.com/open-compass/VLMEvalKit) 🏆
+### Currently, OpenVLM Subjective Leaderboard covers {} different VLMs (including GPT4o, Gemini, Qwen2.5-VL, InternVL2.5 etc.) and {} different multi-modal benchmarks. 
+
+This leaderboard was last updated: {}. 
+
+OpenVLM Subjective Leaderboard only includes open-source VLMs or API models that are publicly available. To add your own model to the leaderboard, please create a PR in [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) to support your VLM and then we will help with the evaluation and updating the leaderboard. For any questions or concerns, please feel free to contact us at [opencompass, fangxinyu, dingshengyuan]@pjlab.org.cn.
+"""
+# CONSTANTS-FIELDS
+META_FIELDS = [
+    'Method', 'Param (B)', 'Language Model', 'Vision Model', 'OpenSource', 'Eval Date'
+]
+MAIN_FIELDS = [
+    'Creation_MMBench', 'MIA-Bench', 'MM-IFEval',
+    'MMAlignBench', 'MMVet', 'WildVision'
+]
+DEFAULT_BENCH = [
+    'Creation_MMBench', 'MIA-Bench', 'MM-IFEval',
+    'MMAlignBench', 'MMVet', 'WildVision'
+]
+MODEL_SIZE = ['<4B', '4B-10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
+MODEL_TYPE = ['API', 'OpenSource']
+
+# The README file for each benchmark
+LEADERBOARD_MD = {}
+
+LEADERBOARD_MD['MAIN'] = f"""
+## Main Evaluation Results
+
+- Metrics:
+  - Avg Score: The average score on all VLM Benchmarks (normalized to 0 - 100, the higher the better). 
+  - Avg Rank: The average rank on all VLM Benchmarks (the lower the better). 
+  - Avg Score & Rank are calculated based on selected benchmark. **When results for some selected benchmarks are missing, Avg Score / Rank will be None!!!** 
+  - Metrics For each dataset
+    - Creation-MMBench: Reward Score/Visual Factuality Score
+    - MMAlignBench, WildVision: Reward Score
+    - MIA-Bench, MM-IFEval, MMVet: Overall Score
+- By default, we present the overall evaluation results based on {len(DEFAULT_BENCH)} VLM benchmarks, sorted by the descending order of Avg Score. 
+  - The following datasets are included in the main results: {', '.join(DEFAULT_BENCH)}. 
+  - Detailed evaluation results for each dataset (included or not included in main) are provided in the consequent tabs. 
+"""
+
+LEADERBOARD_MD['Creation_MMBench'] = """
+## Creation MMBench Evaluation Results
+
+- Creation-MMBench is a multimodal benchmark specifically designed to evaluate the creative capabilities of MLLMs. It consists of **765 test cases**, covering **51 fine-grained tasks** across **4 categories**: *Literary Writing*, *Creative Multimodal Understanding*, *Professional Functionality Writing*, and *Common Functionality Writing*. As an MLLM benchmark, it contains a total of **1001 images** encompassing more than 25 different categories, with some questions incorporating up to 9 images.
+- Creation-MMBench includes carefully crafted **instance-specific criteria** for each test case, enabling assessment of both general response quality and visual-factual alignment in model-generated content. It employs a pair-wise comparison approach, where the model's output is compared with the reference answer (considering both the true answer, input prompt, and visual content) to get the assessment result. The **Dual Evaluation, and GPT-4o judge model** is the evaluation strategy for Creation-MMBench.
+- VFS stands for Visual Factuality Score. The rankings in this leaderboard are arranged in descending order based on the overall reward of each model, with **GPT-4o-1120** providing the **Reference Answer** for comparison, thus serving as the Baseline Model.
+- View More detail on [**Creation-MMBench Official WebPage**](https://open-compass.github.io/Creation-MMBench/)
+"""
+
+LEADERBOARD_MD['MM-IFEval'] = """
+## MM-IFEval Evaluation Results
+
+- MM-IFEval is a comprehensive multimodal instruction-following benchmark designed to rigorously assess the capabilities of Multimodal Large Language Models (MLLMs). It includes 400 high-quality questions across two levels: 300 compose-level tasks that emphasize output format and content constraints, and 100 perception-level tasks that require precise visual understanding.
+- To ensure accurate evaluation, MM-IFEval employs a hybrid strategy combining rule-based verification with LLM-based judgment models. More details see https://arxiv.org/abs/2504.07957
+- Currently, we use GPT4o (gpt-4o-2024-05-13) when needing an LLM judge model.
+"""
+
+LEADERBOARD_MD['MMAlignBench'] = """
+## MMAlignBench Evaluation Results
+
+- MM-AlignBench target for evaluating MLLMs' alignment with human preferences. It includes 252 high-quality, human-annotated samples with diverse image types and open-ended questions. Modeled after Arena-style benchmarks, it uses GPT-4o as the judge model and Claude-Sonnet-3 as the reference model.
+- More Details see https://github.com/PhoenixZ810/OmniAlign-V
+"""
+
+LEADERBOARD_MD['MIA-Bench'] = """
+## MIA-Bench Evaluation Results
+
+- MIA-Bench contains 400 carefully-crafted image-prompt pairs that stress‐test an MLLM’s ability to **follow layered, exacting instructions** in its responses.  ([MIA-Bench: Towards Better Instruction Following Evaluation of Multimodal LLMs](https://arxiv.org/abs/2407.01509), [Towards Better Instruction Following Evaluation of Multimodal LLMs](https://machinelearning.apple.com/research/mia-bench))  
+- The leaderboard reports the **overall avg score**. Judge Model is **GPT-4o**.
+"""
+
+LEADERBOARD_MD['MMVet'] = """
+## MMVet Evaluation Results
+
+- In MMVet Evaluation, we use GPT-4-Turbo (gpt-4-1106-preview) as the judge LLM to assign scores to the VLM outputs. We only perform the evaluation once due to the limited variance among results of multiple evaluation pass originally reported. 
+- No specific prompt template adopted for **ALL VLMs**.
+"""
+
+LEADERBOARD_MD['WildVision'] = """
+## WildVision Evaluation Results
+
+- WildVision-Bench offers **500 real-world multimodal prompts** curated from the WildVision-Arena crowdsourcing platform to benchmark models **by human preference** in natural conversations. 
+- The leaderboard lists reports the **overall reward score**. 
+- Judge Model is **GPT-4o**. Reference Model is **Claude-Sonnet-3**.
+"""

requirements.txt

@@ -0,0 +1,3 @@
+gradio==4.15.0
+numpy>=1.23.4
+pandas>=1.5.3