Hugging Face's logo

Spaces:
TIGER-Lab
/
GenAI-Arena
Running on Zero

App Files Community
18
DongfuJiang commited on
Commit
f6608c4
·
1 Parent(s): 73848fe

update

Browse files
Files changed (11) hide show
  1. arena_elo/elo_rating/clean_battle_data.py +2 -1
  2. arena_elo/generation_model_info.json +6 -1
  3. arena_elo/results/20240315/elo_results_image_editing.pkl +2 -2
  4. arena_elo/results/20240327/clean_battle_t2i_generation.json +0 -0
  5. arena_elo/results/20240327/elo_results_t2i_generation.pkl +2 -2
  6. arena_elo/results/20240327/t2i_generation_leaderboard.csv +9 -10
  7. arena_elo/results/latest/clean_battle_t2i_generation_20240327.json +0 -0
  8. arena_elo/results/latest/elo_results_image_editing.pkl +2 -2
  9. arena_elo/results/latest/elo_results_t2i_generation.pkl +2 -2
  10. arena_elo/results/latest/t2i_generation_leaderboard.csv +9 -10
  11. arena_elo/update_elo_rating.sh +2 -2
arena_elo/elo_rating/clean_battle_data.py CHANGED
@@ -82,6 +82,7 @@ def replace_model_name(old_name, tstamp):
82
  "claude-instant-v1": "claude-instant-1",
83
  "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
84
  "claude-2": "claude-2.0",
 
85
  }
86
  if old_name in ["gpt-4", "gpt-3.5-turbo"]:
87
  if tstamp > 1687849200:
@@ -203,7 +204,6 @@ def clean_battle_data(
203
  # continue
204
 
205
  # Replace bard with palm
206
- models = [replace_model_name(m, row["tstamp"]) for m in models]
207
  if task_name == "image_editing":
208
  if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
209
  # print(f"Invalid model names: {models}")
@@ -222,6 +222,7 @@ def clean_battle_data(
222
 
223
  else:
224
  raise ValueError(f"Invalid task_name: {task_name}")
 
225
 
226
  # Exclude certain models
227
  if exclude_model_names and any(x in exclude_model_names for x in models):
 
82
  "claude-instant-v1": "claude-instant-1",
83
  "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
84
  "claude-2": "claude-2.0",
85
+ "PlayGroundV2": "Playground v2",
86
  }
87
  if old_name in ["gpt-4", "gpt-3.5-turbo"]:
88
  if tstamp > 1687849200:
 
204
  # continue
205
 
206
  # Replace bard with palm
 
207
  if task_name == "image_editing":
208
  if not all(x.startswith("imagenhub_") and x.endswith("_edition") for x in models):
209
  # print(f"Invalid model names: {models}")
 
222
 
223
  else:
224
  raise ValueError(f"Invalid task_name: {task_name}")
225
+ models = [replace_model_name(m, row["tstamp"]) for m in models]
226
 
227
  # Exclude certain models
228
  if exclude_model_names and any(x in exclude_model_names for x in models):
arena_elo/generation_model_info.json CHANGED
@@ -9,6 +9,11 @@
9
  "License": "Playground v2 Community License",
10
  "Organization": "Playground"
11
  },
 
 
 
 
 
12
  "OpenJourney": {
13
  "Link": "https://huggingface.co/prompthero/openjourney",
14
  "License": "creativeml-openrail-m",
@@ -39,4 +44,4 @@
39
  "License": "stable-cascade-nc-community (other)",
40
  "Organization": "Stability AI"
41
  }
42
- }
 
9
  "License": "Playground v2 Community License",
10
  "Organization": "Playground"
11
  },
12
+ "Playground v2.5": {
13
+ "Link": "https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic",
14
+ "License": "Playground v2 Community License",
15
+ "Organization": "Playground"
16
+ },
17
  "OpenJourney": {
18
  "Link": "https://huggingface.co/prompthero/openjourney",
19
  "License": "creativeml-openrail-m",
 
44
  "License": "stable-cascade-nc-community (other)",
45
  "Organization": "Stability AI"
46
  }
47
+ }
arena_elo/results/20240315/elo_results_image_editing.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e528d30840c8a5787b0d2f08f27758b02f7eb718ccab695010b30df2127efe5e
3
- size 57064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
3
+ size 57091
arena_elo/results/20240327/clean_battle_t2i_generation.json CHANGED
The diff for this file is too large to render. See raw diff
 
arena_elo/results/20240327/elo_results_t2i_generation.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fec01fe5af62dce3990634cffd1d926330ccbf170ef0c3b5d2f07fb06c4cf149
3
- size 65189
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
3
+ size 62414
arena_elo/results/20240327/t2i_generation_leaderboard.csv CHANGED
@@ -1,11 +1,10 @@
1
  key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
2
- Playground v2.5,Playground v2.5,1212.4660228554317,1233.021110469063,N/A,N/A,N/A
3
- StableCascade,StableCascade,1098.8180832734447,1081.4707812969855,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
4
- PlayGroundV2,PlayGroundV2,1089.993871580802,1088.6262085724481,N/A,N/A,N/A
5
- Playground v2,Playground v2,1049.6156124554975,1051.618375116693,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
6
- SDXLLightning,SDXLLightning,1036.8582186059539,1039.3079223370821,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
7
- PixArtAlpha,PixArtAlpha,1016.2085497703334,1002.5100184720693,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
8
- SDXL,SDXL,960.5073412035289,965.3037978455568,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
9
- SDXLTurbo,SDXLTurbo,907.997473382927,910.1644152252661,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
10
- OpenJourney,OpenJourney,836.9689192463355,827.9470053715127,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
11
- LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
 
1
  key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
2
+ Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
3
+ StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
4
+ Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
5
+ SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
6
+ PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
7
+ SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
8
+ SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
9
+ OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
10
+ LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
 
arena_elo/results/latest/clean_battle_t2i_generation_20240327.json CHANGED
The diff for this file is too large to render. See raw diff
 
arena_elo/results/latest/elo_results_image_editing.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e528d30840c8a5787b0d2f08f27758b02f7eb718ccab695010b30df2127efe5e
3
- size 57064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cef00c45d392a30913b367825270fcee5fd29e5c830866eef3d07146b3502f3
3
+ size 57091
arena_elo/results/latest/elo_results_t2i_generation.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fec01fe5af62dce3990634cffd1d926330ccbf170ef0c3b5d2f07fb06c4cf149
3
- size 65189
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f525abe69feb822d341929b27ef7660ddd5e6ff0491bed8383a8e3d19f0342bd
3
+ size 62414
arena_elo/results/latest/t2i_generation_leaderboard.csv CHANGED
@@ -1,11 +1,10 @@
1
  key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
2
- Playground v2.5,Playground v2.5,1212.4660228554317,1233.021110469063,N/A,N/A,N/A
3
- StableCascade,StableCascade,1098.8180832734447,1081.4707812969855,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
4
- PlayGroundV2,PlayGroundV2,1089.993871580802,1088.6262085724481,N/A,N/A,N/A
5
- Playground v2,Playground v2,1049.6156124554975,1051.618375116693,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
6
- SDXLLightning,SDXLLightning,1036.8582186059539,1039.3079223370821,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
7
- PixArtAlpha,PixArtAlpha,1016.2085497703334,1002.5100184720693,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
8
- SDXL,SDXL,960.5073412035289,965.3037978455568,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
9
- SDXLTurbo,SDXLTurbo,907.997473382927,910.1644152252661,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
10
- OpenJourney,OpenJourney,836.9689192463355,827.9470053715127,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
11
- LCM,LCM,790.5659076257482,805.8155782210948,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
 
1
  key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
2
+ Playground v2.5,Playground v2.5,1226.2872445351936,1246.1685934024742,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic
3
+ StableCascade,StableCascade,1105.3322734027522,1087.9198960927265,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
4
+ Playground v2,Playground v2,1091.4371447234744,1090.676108819673,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
5
+ SDXLLightning,SDXLLightning,1043.235902888147,1045.0529259890538,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
6
+ PixArtAlpha,PixArtAlpha,1020.6412075829058,1006.9966036187151,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
7
+ SDXL,SDXL,964.7626495363717,969.5241392802999,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
8
+ SDXLTurbo,SDXLTurbo,912.2113859675355,914.3805456579931,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
9
+ OpenJourney,OpenJourney,841.2224045541894,832.2282703082603,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
10
+ LCM,LCM,794.8697868094328,810.2118373597045,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
 
arena_elo/update_elo_rating.sh CHANGED
@@ -22,10 +22,10 @@ mv clean_battle_image_editing_$edition_battle_cutoff_date.json ./results/$editio
22
  mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
23
 
24
 
25
- python3 -m elo_rating.elo_analysis --clean-battle-file clean_battle_image_editing_$edition_battle_cutoff_date.json
26
  mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
27
 
28
- python3 -m elo_rating.elo_analysis --clean-battle-file clean_battle_t2i_generation_$generation_battle_cutoff_date.json
29
  mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
30
 
31
  # generat the leaderboard
 
22
  mv clean_battle_t2i_generation_$generation_battle_cutoff_date.json ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
23
 
24
 
25
+ python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$edition_battle_cutoff_date/clean_battle_image_editing.json
26
  mv ./elo_results_$edition_battle_cutoff_date.pkl ./results/$edition_battle_cutoff_date/elo_results_image_editing.pkl
27
 
28
+ python3 -m elo_rating.elo_analysis --clean-battle-file ./results/$generation_battle_cutoff_date/clean_battle_t2i_generation.json
29
  mv ./elo_results_$generation_battle_cutoff_date.pkl ./results/$generation_battle_cutoff_date/elo_results_t2i_generation.pkl
30
 
31
  # generat the leaderboard