diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..4757fc72cdd035b666e98c371a99a93801e2da93 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..095325ae0338009f1a74d6492d66f84c72aaefae
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*$py.class
+.env
+.venv
+env/
+venv/
+ENV/
+.DS_Store
+*.log
+data/data_viewer.jsonl
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..20ed019d3b9f276f29e708c3b435e545d2a36888
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,11 @@
+FROM python:3.10-slim
+
+WORKDIR /code
+
+COPY . /code/
+
+RUN pip install --no-cache-dir --upgrade pip && \
+ pip install --no-cache-dir -r requirements.txt
+
+# 默认运行命令
+CMD ["python", "app.py"]
\ No newline at end of file
diff --git a/README.md b/README.md
index 6a0910a0db9c51b46763c29bce9e5836e8585913..6aa28a0bd2d16e1a3719b9ae7f1da8f86ce8e975 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,25 @@
----
-title: DeepResearch Bench
-emoji: 🏆
-colorFrom: green
-colorTo: red
-sdk: gradio
-sdk_version: 5.31.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DeepResearch Bench
+
+**DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents**
+
+This application showcases comprehensive evaluation results for Deep Research Agents. The app includes:
+
+- 🏆 **Leaderboard** - View overall performance metrics across all evaluated models
+- 🔍 **Data Viewer** - Explore detailed results for individual research tasks
+- 📊 **Side-by-Side Comparison** - Compare different models' responses to the same research questions
+
+Visit our [project website](https://deepresearch-bench.github.io) for more information.
+
+## Citation
+```bibtex
+@article{du2025deepresearch,
+ author = {Mingxuan Du and Benfeng Xu and Chiwei Zhu and Xiaorui Wang and Zhendong Mao},
+ title = {DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents},
+ journal = {arXiv preprint},
+ year = {2025},
+}
+```
+
+## Hugging Face Space Details
+- SDK: Gradio
+- SDK Version: 3.50.0
\ No newline at end of file
diff --git a/__pycache__/create_leaderboard.cpython-38.pyc b/__pycache__/create_leaderboard.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3408c5a29d1c6283bc166e8421f6005cbc232be4
Binary files /dev/null and b/__pycache__/create_leaderboard.cpython-38.pyc differ
diff --git a/__pycache__/create_leaderboard.cpython-39.pyc b/__pycache__/create_leaderboard.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1cf01a49dc75d663c9dada97111764009bfe2e1b
Binary files /dev/null and b/__pycache__/create_leaderboard.cpython-39.pyc differ
diff --git a/__pycache__/gradio.cpython-310.pyc b/__pycache__/gradio.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3a14d699f12e8124a5a65a6c832e0edd38943945
Binary files /dev/null and b/__pycache__/gradio.cpython-310.pyc differ
diff --git a/__pycache__/gradio.cpython-39.pyc b/__pycache__/gradio.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ff2aea33cdd9d956bbcb684ce606e41faf533a6d
Binary files /dev/null and b/__pycache__/gradio.cpython-39.pyc differ
diff --git a/app.py b/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa612e18bbb18e39e9662df90f2d194a714bd84b
--- /dev/null
+++ b/app.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+DeepResearch Bench HF Space 入口文件
+"""
+
+from __future__ import annotations
+from create_leaderboard import demo
+
+# 在Hugging Face Space中运行
+if __name__ == "__main__":
+ demo.launch(
+ server_name="0.0.0.0", # 必须这样设置以允许外部访问
+ share=False, # HF Space 自己有分享功能，无需额外分享
+ show_api=False, # 隐藏API文档页面
+ )
\ No newline at end of file
diff --git a/create_leaderboard.py b/create_leaderboard.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c6003fe69d529d74590f76ec4f234d20015602c
--- /dev/null
+++ b/create_leaderboard.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Gradio UI – v2.1 (Leaderboard · Data Viewer · Prompt-to-Leaderboard)
+"""
+
+from __future__ import annotations
+from pathlib import Path
+import gradio as gr
+
+# ---- Tab 组件 ----
+from tabs.leaderboard_tab import create_leaderboard_tab
+from tabs.data_viewer_tab import create_data_viewer_tab
+from tabs.data_viewer_side_by_side_tab import create_data_viewer_side_by_side_tab
+
+# ---------------------------------------------------------------------------
+# UI
+# ---------------------------------------------------------------------------
+
+with gr.Blocks(title="DeepResearch Bench") as demo:
+
+ # ========= 全局 CSS（仅作用于自定义标题 & 简介） =========
+ gr.HTML("""
+
+ """)
+
+ # ========= 顶部标题 & 简介（不使用 Markdown 标题语法） =========
+ gr.HTML("""
+
+ DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents
+
+
+
+ The research aims to comprehensively evaluate the capabilities of Deep Research Agents.
+
Code |
+
Website |
+
Paper |
+
Eval Dataset |
+ Total models: 16 | Last Update: 28 May 2025
+
siveness", comprehensiveness),
+ ("Insight
Score", insight),
+ ("Instruction
Following", instruction),
+ ("Readability
Score", readability)
+ ]
+ html_items_str = ""
+ for title, score in scores_data:
+ score_value = score if score is not None else "N/A"
+ html_items_str += f"""
+
+
{title}
+
{score_value}
+
{title}
+ {score_value}
+
+ """
+
+ # Outer container styled to mimic the .card class from the main CSS block
+ return f"""
+"""
+
+# ---------- 生成 Tab ----------
+def create_data_viewer_tab():
+ with gr.Tab("🔍Data Viewer"):
+ gr.HTML(
+ """
+
+"""
+ )
+
+ df = load_data_viewer_data()
+ if df.empty:
+ gr.Markdown("## ⚠️ 没有可用数据 \n请确认 `data/data_viewer.jsonl` 存在且字段齐全(包括所有分数)。")
+ return
+
+ models = sorted(df["model_name"].unique())
+ tasks_df = (
+ df[["id", "prompt"]].drop_duplicates()
+ .assign(id_num=lambda x: x["id"].astype(int))
+ .sort_values("id_num")
+ )
+
+ task_choices = []
+ for _, row in tasks_df.iterrows():
+ limit = 30 if int(row["id"]) <= 50 else 60
+ preview = row["prompt"][:limit] + ("…" if len(row["prompt"]) > limit else "")
+ task_choices.append(f"{row['id']}. {preview}")
+
+ init_model = random.choice(models) if models else None
+ init_task = random.choice(task_choices) if task_choices else None
+
+ with gr.Row():
+ model_dd = gr.Dropdown(label="Select Model", choices=models, value=init_model, interactive=True)
+ task_dd = gr.Dropdown(label="Select Task", choices=task_choices, value=init_task, interactive=True)
+
+ user_md = gr.Markdown(elem_classes=["card", "scrollable-sm"])
+ article_md = gr.Markdown(elem_classes=["card", "scrollable-lg"])
+ scores_html = gr.HTML() # 新增HTML组件用于显示分数
+
+ def fetch(model, task_disp):
+ if not model or not task_disp:
+ msg = "请选择模型和任务。"
+ return make_user_task_markdown("--", msg), make_article_markdown(msg), ""
+
+ item_id = task_disp.split(".", 1)[0].strip()
+ entry = df[(df["model_name"] == model) & (df["id"] == item_id)]
+ if entry.empty:
+ err = f"未找到模型 **{model}** 对应任务 **{item_id}** 的内容或分数。"
+ return make_user_task_markdown(item_id, err), make_article_markdown(err), ""
+
+ prompt = entry["prompt"].iloc[0]
+ article = entry["article"].iloc[0]
+
+ # 提取分数
+ overall = entry["overall_score"].iloc[0]
+ comprehensiveness = entry["comprehensiveness_score"].iloc[0]
+ insight = entry["insight_score"].iloc[0]
+ instruction = entry["instruction_following_score"].iloc[0]
+ readability = entry["readability_score"].iloc[0]
+
+ scores_content = make_scores_html(overall, comprehensiveness, insight, instruction, readability)
+
+ return make_user_task_markdown(item_id, prompt), make_article_markdown(article), scores_content
+
+ # 初始渲染
+ if init_model and init_task:
+ user_md.value, article_md.value, scores_html.value = fetch(init_model, init_task)
+ else:
+ user_md.value = make_user_task_markdown("--", "请选择模型和任务。")
+ article_md.value = make_article_markdown("请选择模型和任务。")
+ scores_html.value = ""
+
+ model_dd.change(fetch, inputs=[model_dd, task_dd], outputs=[user_md, article_md, scores_html])
+ task_dd.change(fetch, inputs=[model_dd, task_dd], outputs=[user_md, article_md, scores_html])
\ No newline at end of file
diff --git a/tabs/leaderboard_tab.py b/tabs/leaderboard_tab.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fe0876846b97fdf972dce3c6cbdded86db6cfea
--- /dev/null
+++ b/tabs/leaderboard_tab.py
@@ -0,0 +1,176 @@
+import gradio as gr
+import pandas as pd
+from pathlib import Path
+
+# 相对于主脚本的路径调整
+BASE_DIR = Path(__file__).resolve().parent.parent
+DATA_PATH = BASE_DIR / "data" / "leaderboard.csv"
+
+# 新增：用于高亮显示的常量
+HIGHLIGHT_COLOR = "#E6D8FF"
+CATEGORY_TO_HIGHLIGHT = "Deep Research Agent"
+
+# 新增：列名重命名映射
+COLUMN_RENAME_MAP = {
+ 'overall_score': 'overall',
+ 'comprehensiveness': 'comp.',
+ 'insight': 'insight',
+ 'instruction_following': 'inst.',
+ 'readability': 'read.',
+ 'citation_accuracy': 'c.acc.',
+ 'effective_citations': 'eff.c.'
+}
+
+# 模型分类映射
+MODEL_CATEGORIES = {
+ "Deep Research Agent": [
+ "gemini-2.5-pro-deepresearch",
+ "grok-deeper-search",
+ "openai-deepresearch",
+ "perplexity-Research"
+ ],
+ "LLM with Search": [
+ "claude-3-7-sonnet-with-search",
+ "perplexity-sonar-reasoning-pro",
+ "perplexity-sonar-reasoning",
+ "perplexity-sonar-pro",
+ "gemini-2.5-pro-with-grounding",
+ "gpt-4o-search-preview",
+ "perplexity-sonar",
+ "gpt-4.1-with-search",
+ "gemini-2.5-flash-preview-04-17",
+ "gpt-4o-mini-search-preview",
+ "gpt-4.1-mini-with-search",
+ "claude-3-5-sonnet-with-search"
+ ]
+}
+
+def load_leaderboard() -> pd.DataFrame:
+ if not DATA_PATH.exists():
+ raise FileNotFoundError(
+ f"Leaderboard file not found: {DATA_PATH}.\n"
+ "→ 先运行 rank_leaderboard.py 生成 data/leaderboard.csv"
+ )
+ df = pd.read_csv(DATA_PATH)
+ df.columns = [c.strip() for c in df.columns]
+
+ def get_category(model_name):
+ for category, models in MODEL_CATEGORIES.items():
+ if model_name in models:
+ return category
+ return "Others" # 为不在预定义类别中的模型提供默认类别
+
+ df['category'] = df['model'].apply(get_category)
+ return df
+
+def make_ranked(df: pd.DataFrame) -> pd.DataFrame:
+ ranked = df.sort_values(by='overall_score', ascending=False).reset_index(drop=True)
+ ranked.insert(0, "Rank", range(1, len(ranked) + 1))
+
+ # 重命名列名为简写形式
+ ranked = ranked.rename(columns=COLUMN_RENAME_MAP)
+
+ return ranked
+
+def filter_data(search_text: str, selected_categories: list):
+ df = load_leaderboard()
+
+ if search_text.strip():
+ df = df[df['model'].str.contains(search_text.strip(), case=False, na=False)]
+
+ if selected_categories:
+ df = df[df['category'].isin(selected_categories)]
+
+ return make_ranked(df)
+
+# 新增：辅助函数用于样式化DataFrame
+def _style_specific_rows(row, category_column_name='category', target_category=CATEGORY_TO_HIGHLIGHT, color=HIGHLIGHT_COLOR):
+ """
+ 根据行的类别返回样式列表。如果类别匹配目标类别，则应用背景色。
+ """
+ apply_color = color if row.get(category_column_name) == target_category else ''
+ return [f'background-color: {apply_color}' for _ in row]
+
+def _apply_table_styling(df: pd.DataFrame) -> pd.io.formats.style.Styler:
+ """
+ 应用表格样式：
+ - 高亮显示 CATEGORY_TO_HIGHLIGHT 的行
+ - 保留 'category' 列显示
+ - 格式化数值为两位小数
+ 返回 Pandas Styler 对象。
+ """
+ if df.empty:
+ return df.style
+
+ styled_df = df.copy()
+
+ # 获取数值列（排除 Rank, model, category 列）
+ numeric_columns = []
+ for col in styled_df.columns:
+ if col not in ['Rank', 'model', 'category']:
+ # 检查是否为数值类型
+ if styled_df[col].dtype in ['float64', 'int64'] or pd.api.types.is_numeric_dtype(styled_df[col]):
+ numeric_columns.append(col)
+
+ # 应用行样式 - 高亮特定类别的行
+ styler = styled_df.style.apply(
+ _style_specific_rows,
+ axis=1,
+ category_column_name='category',
+ target_category=CATEGORY_TO_HIGHLIGHT,
+ color=HIGHLIGHT_COLOR
+ )
+
+ # 使用 Styler 的 format 方法格式化数值列为两位小数
+ if numeric_columns:
+ format_dict = {col: '{:.2f}' for col in numeric_columns}
+ styler = styler.format(format_dict)
+
+ return styler
+
+def create_leaderboard_tab():
+ with gr.Tab("🏆Leaderboard"):
+ with gr.Row():
+ search_box = gr.Textbox(
+ label="Model Search",
+ placeholder="Entering model name to search...",
+ value=""
+ )
+ category_checkboxes = gr.CheckboxGroup(
+ label="Model Categories",
+ choices=list(MODEL_CATEGORIES.keys()),
+ value=list(MODEL_CATEGORIES.keys())
+ )
+
+ initial_df_raw = make_ranked(load_leaderboard())
+ styled_initial_value = _apply_table_styling(initial_df_raw.copy())
+
+ table = gr.Dataframe(
+ interactive=False,
+ wrap=False,
+ value=styled_initial_value,
+ )
+
+ def update_display(search_text, selected_categories):
+ filtered_df_raw = filter_data(search_text, selected_categories)
+ styled_updated_value = _apply_table_styling(filtered_df_raw.copy())
+ return styled_updated_value
+
+ search_box.change(
+ fn=update_display,
+ inputs=[search_box, category_checkboxes],
+ outputs=table
+ )
+ category_checkboxes.change(
+ fn=update_display,
+ inputs=[search_box, category_checkboxes],
+ outputs=table
+ )
+
+ # 在底部添加列名说明
+ gr.Markdown("""
+ ### Column Abbreviations
+ The leaderboard uses abbreviated column names for compact display: (i) **overall** - Overall Score; (ii) **comp.** - Comprehensiveness; (iii) **insight** - Insight quality; (iv) **inst.** - Instruction Following; (v) **read.** - Readability; (vi) **c.acc.** - Citation Accuracy; (vii) **eff.c.** - Effective Citations.
+ """)
+
+ return search_box
\ No newline at end of file
diff --git a/utils/merge_raw_data.py b/utils/merge_raw_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c46dfc8762cda0ed8af5d4801a359f09860b844
--- /dev/null
+++ b/utils/merge_raw_data.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+import os
+from pathlib import Path
+
+
+def calculate_dimension_score(target_score, reference_score):
+ """计算单个维度的分数，与rank_leaderboard.py中的逻辑一致"""
+ if (target_score + reference_score) == 0: # 避免除以零
+ return 0.0
+ return target_score / (target_score + reference_score)
+
+
+def load_scores_for_model(model_results_file_path: Path):
+ """为单个模型加载所有文章的评分数据"""
+ scores_by_id = {}
+ if not model_results_file_path.exists():
+ print(f"警告: 未找到模型 {model_results_file_path.stem} 的结果文件: {model_results_file_path}")
+ return scores_by_id
+
+ print(f" 正在从 {model_results_file_path.name} 加载分数...")
+ with open(model_results_file_path, 'r', encoding='utf-8') as f:
+ for i, line in enumerate(f):
+ try:
+ data = json.loads(line.strip())
+ article_id = str(data.get('id')) # 确保ID是字符串以供匹配
+ if not article_id:
+ print(f" 警告: {model_results_file_path.name} 第 {i+1} 行缺少ID，已跳过。")
+ continue
+
+ # 直接获取 overall_score (原始值，假设在0-1范围，或者已经是0-100，根据您的数据调整)
+ # 根据您之前的修改，这里我们假设原始overall_score需要乘以100
+ overall_score_raw = data.get('overall_score', 0.0)
+ overall_score_scaled = overall_score_raw * 100
+
+ # 计算四个维度的分数
+ comp_score_raw = calculate_dimension_score(
+ data.get('target_comprehensiveness_weighted_avg', 0),
+ data.get('reference_comprehensiveness_weighted_avg', 0)
+ )
+ insight_score_raw = calculate_dimension_score(
+ data.get('target_insight_weighted_avg', 0),
+ data.get('reference_insight_weighted_avg', 0)
+ )
+ instruction_score_raw = calculate_dimension_score(
+ data.get('target_instruction_following_weighted_avg', 0),
+ data.get('reference_instruction_following_weighted_avg', 0)
+ )
+ readability_score_raw = calculate_dimension_score(
+ data.get('target_readability_weighted_avg', 0),
+ data.get('reference_readability_weighted_avg', 0)
+ )
+
+ scores_by_id[article_id] = {
+ 'overall_score': f"{overall_score_scaled:.2f}",
+ 'comprehensiveness_score': f"{comp_score_raw * 100:.2f}",
+ 'insight_score': f"{insight_score_raw * 100:.2f}",
+ 'instruction_following_score': f"{instruction_score_raw * 100:.2f}",
+ 'readability_score': f"{readability_score_raw * 100:.2f}"
+ }
+ except json.JSONDecodeError as e:
+ print(f" 错误: 解析JSON时出错 (文件: {model_results_file_path.name}, 行: {i+1}): {e}")
+ except Exception as e:
+ print(f" 错误: 处理数据时出错 (文件: {model_results_file_path.name}, 行: {i+1}): {e}")
+ print(f" 为模型 {model_results_file_path.stem} 加载了 {len(scores_by_id)}篇文章的分数")
+ return scores_by_id
+
+
+def merge_jsonl_files():
+ # 定义目录路径
+ project_root = Path(__file__).resolve().parent.parent
+ raw_data_dir = project_root / "data" / "raw_data" # 包含原始文章内容的目录
+ raw_results_dir = project_root / "data" / "raw_results" # 包含评分结果的目录
+ output_file = project_root / "data" / "data_viewer.jsonl"
+
+ # 获取所有原始数据JSONL文件
+ input_files = list(raw_data_dir.glob("*.jsonl"))
+ print(f"在 {raw_data_dir} 中找到 {len(input_files)} 个模型JSONL文件")
+
+ if not input_files:
+ print("未找到任何原始数据文件，已退出。")
+ return
+
+ # 清空输出文件 (如果需要，或者可以采用追加模式，但通常合并操作会重新生成)
+ with open(output_file, 'w', encoding='utf-8') as f:
+ pass # 创建或清空文件
+
+ all_merged_data = []
+
+ for raw_data_file in input_files:
+ model_name = raw_data_file.stem
+ print(f"正在处理原始数据文件: {raw_data_file.name} (模型: {model_name})")
+
+ # 为当前模型加载评分数据
+ model_results_file = raw_results_dir / f"{model_name}.jsonl"
+ scores_for_current_model = load_scores_for_model(model_results_file)
+
+ processed_articles_count = 0
+ with open(raw_data_file, 'r', encoding='utf-8') as f_raw:
+ for i, line in enumerate(f_raw):
+ try:
+ article_data = json.loads(line.strip())
+ article_id = str(article_data.get('id')) # 确保ID是字符串
+
+ if not article_id:
+ print(f" 警告: {raw_data_file.name} 第 {i+1} 行缺少ID，已跳过。")
+ continue
+
+ # 从加载的评分数据中获取该文章的评分
+ article_scores = scores_for_current_model.get(article_id, {})
+ if not article_scores:
+ print(f" 警告: 模型 {model_name} 的文章ID {article_id} 未在结果文件中找到分数。")
+
+ merged_item = {
+ 'model_name': model_name,
+ 'id': article_id,
+ 'prompt': article_data.get('prompt'),
+ 'article': article_data.get('article'),
+ 'overall_score': article_scores.get('overall_score'), # 可能为None
+ 'comprehensiveness_score': article_scores.get('comprehensiveness_score'),
+ 'insight_score': article_scores.get('insight_score'),
+ 'instruction_following_score': article_scores.get('instruction_following_score'),
+ 'readability_score': article_scores.get('readability_score')
+ }
+ all_merged_data.append(merged_item)
+ processed_articles_count += 1
+ except json.JSONDecodeError as e:
+ print(f" 错误: 解析原始数据JSON时出错 (文件: {raw_data_file.name}, 行: {i+1}): {e}")
+ except Exception as e:
+ print(f" 错误: 处理原始数据时出错 (文件: {raw_data_file.name}, 行: {i+1}): {e}")
+ print(f" 为模型 {model_name} 处理了 {processed_articles_count} 篇文章数据。")
+
+ # 一次性写入所有合并后的数据
+ with open(output_file, 'w', encoding='utf-8') as f_out:
+ for item in all_merged_data:
+ f_out.write(json.dumps(item, ensure_ascii=False) + '\n')
+
+ print(f"\n成功合并并保存到: {output_file}, 共 {len(all_merged_data)} 条记录")
+
+if __name__ == "__main__":
+ merge_jsonl_files()
+ print("所有文件处理完成！")
\ No newline at end of file
diff --git a/utils/rank_leaderboard.py b/utils/rank_leaderboard.py
new file mode 100644
index 0000000000000000000000000000000000000000..86a2754e949f85cb23d98af5593d2d7beada91ce
--- /dev/null
+++ b/utils/rank_leaderboard.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import csv
+from pathlib import Path
+from collections import defaultdict
+
+
+def calculate_dimension_score(target_score, reference_score):
+ """计算单个维度的分数"""
+ return target_score / (target_score + reference_score)
+
+
+def process_model_data(model_file):
+ """处理单个模型文件的数据"""
+ model_name = model_file.stem
+ print(f"正在处理模型: {model_name}")
+
+ overall_scores = []
+ comprehensiveness_scores = []
+ insight_scores = []
+ instruction_following_scores = []
+ readability_scores = []
+
+ with open(model_file, 'r', encoding='utf-8') as f:
+ for line in f:
+ try:
+ data = json.loads(line.strip())
+
+ # 获取总分
+ overall_score = data.get('overall_score', 0)
+ overall_scores.append(overall_score)
+
+ # 计算四个维度的分数
+ target_comp = data.get('target_comprehensiveness_weighted_avg', 0)
+ ref_comp = data.get('reference_comprehensiveness_weighted_avg', 0)
+ comp_score = calculate_dimension_score(target_comp, ref_comp)
+ comprehensiveness_scores.append(comp_score)
+
+ target_insight = data.get('target_insight_weighted_avg', 0)
+ ref_insight = data.get('reference_insight_weighted_avg', 0)
+ insight_score = calculate_dimension_score(target_insight, ref_insight)
+ insight_scores.append(insight_score)
+
+ target_instruction = data.get('target_instruction_following_weighted_avg', 0)
+ ref_instruction = data.get('reference_instruction_following_weighted_avg', 0)
+ instruction_score = calculate_dimension_score(target_instruction, ref_instruction)
+ instruction_following_scores.append(instruction_score)
+
+ target_readability = data.get('target_readability_weighted_avg', 0)
+ ref_readability = data.get('reference_readability_weighted_avg', 0)
+ readability_score = calculate_dimension_score(target_readability, ref_readability)
+ readability_scores.append(readability_score)
+
+ except json.JSONDecodeError as e:
+ print(f"解析JSON时出错 (模型: {model_name}): {e}")
+ continue
+ except Exception as e:
+ print(f"处理数据时出错 (模型: {model_name}): {e}")
+ continue
+
+ # 计算平均分
+ avg_overall = sum(overall_scores) / len(overall_scores)
+ avg_comprehensiveness = sum(comprehensiveness_scores) / len(comprehensiveness_scores)
+ avg_insight = sum(insight_scores) / len(insight_scores)
+ avg_instruction_following = sum(instruction_following_scores) / len(instruction_following_scores)
+ avg_readability = sum(readability_scores) / len(readability_scores)
+ print(f" - 处理了 {len(overall_scores)} 条记录")
+ print(f" - 总分: {avg_overall:.4f}")
+
+ return {
+ 'model': model_name,
+ 'overall_score': avg_overall * 100,
+ 'comprehensiveness': avg_comprehensiveness * 100,
+ 'insight': avg_insight * 100,
+ 'instruction_following': avg_instruction_following * 100,
+ 'readability': avg_readability * 100
+ }
+
+
+def rank_leaderboard():
+ """计算排行榜并保存到CSV"""
+ # 定义目录路径
+ project_root = Path(__file__).parent.parent
+ input_dir = project_root / "data" / "raw_results"
+ output_file = project_root / "data" / "leaderboard.csv"
+
+ # 获取所有JSONL文件
+ input_files = list(input_dir.glob("*.jsonl"))
+ print(f"找到 {len(input_files)} 个模型结果文件")
+
+ if not input_files:
+ print("未找到任何JSONL文件")
+ return
+
+ # 处理每个模型的数据
+ model_results = []
+ for input_file in input_files:
+ try:
+ result = process_model_data(input_file)
+ model_results.append(result)
+ except Exception as e:
+ print(f"处理文件 {input_file.name} 时出错: {e}")
+ continue
+
+ # 按总分排序（降序）
+ model_results.sort(key=lambda x: x['overall_score'], reverse=True)
+
+ # 写入CSV文件
+ with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
+ fieldnames = ['model', 'overall_score', 'comprehensiveness', 'insight', 'instruction_following', 'readability']
+ writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+ # 写入表头
+ writer.writeheader()
+
+ # 写入数据
+ for result in model_results:
+ writer.writerow({
+ 'model': result['model'],
+ 'overall_score': f"{result['overall_score']:.2f}",
+ 'comprehensiveness': f"{result['comprehensiveness']:.2f}",
+ 'insight': f"{result['insight']:.2f}",
+ 'instruction_following': f"{result['instruction_following']:.2f}",
+ 'readability': f"{result['readability']:.2f}"
+ })
+
+ print(f"\n排行榜已保存到: {output_file}")
+ print(f"共处理了 {len(model_results)} 个模型")
+
+
+if __name__ == "__main__":
+ rank_leaderboard()
+ print("排行榜计算完成！")