update

arena_elo/elo_rating/clean_battle_data.py

@@ -82,7 +82,8 @@ def replace_model_name(old_name, tstamp):
         "claude-instant-v1": "claude-instant-1",
         "oasst-sft-1-pythia-12b": "oasst-pythia-12b",
         "claude-2": "claude-2.0",
-        "PlayGroundV2": "Playground v2",
+        "PlayGroundV2": "PlayGround V2",
+        "PlayGroundV2.5": "PlayGround V2.5",
     }
     if old_name in ["gpt-4", "gpt-3.5-turbo"]:
         if tstamp > 1687849200:
@@ -250,7 +251,7 @@ def clean_battle_data(
                     print(f"Invalid model names: {_model}")
                     valid = False
                     break
-                if not (platform in ["playground", "imagenhub"] and task == "generation"):
+                if not (platform.lower() in ["playground", "imagenhub"] and task == "generation"):
                     valid = False
                     break
             if not valid:

arena_elo/elo_rating/generate_leaderboard.py

@@ -30,6 +30,7 @@ def main(
             model_info[model]["License"] = "N/A"
             model_info[model]["Organization"] = "N/A"
             model_info[model]["Link"] = "N/A"
+            print(f"Model {model} not found in model_info.json")
         model_info[model]["Model"] = model
         model_info[model]["key"] = model
 

arena_elo/generation_model_info.json

@@ -9,7 +9,7 @@
         "License": "Playground v2 Community License",
         "Organization": "Playground"
     },
-    "PlaygroundV2.5": {
+    "PlayGroundV2.5": {
         "Link": "https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic",
         "License": "Playground v2.5 Community License",
         "Organization": "Playground"

arena_elo/results/20240411/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e2e2b693cc44a7ba8b8abf1501f3f1ebda6c232b5ee79f760545705a7ca3248
-size 62419
+oid sha256:f923aeeaecb985cafb5ec70c59daef664f625b39bffc38a488e8066c701e1146
+size 62428

arena_elo/results/20240411/t2i_generation_leaderboard.csv

@@ -1,10 +1,10 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGroundV2.5,PlayGroundV2.5,1226.9110139608943,1102.8780955386826,N/A,N/A,N/A
-StableCascade,StableCascade,1105.4281747413283,1059.3744882854992,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1090.337952005879,1100.0232585639621,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1042.9663858069512,1039.7571032040712,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6589616543491,1022.4559565308926,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,965.437822756589,999.8413183312834,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2388336601439,950.6036749825616,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.1845129591121,873.4879879328548,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8363424547509,852.4223549305145,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1226.9110139608943,1105.1039810421305,N/A,N/A,N/A
+StableCascade,StableCascade,1105.4281747413283,1059.2855612093028,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+PlayGround V2,PlayGround V2,1090.337952005879,1099.3544589205903,N/A,N/A,N/A
+SDXLLightning,SDXLLightning,1042.9663858069512,1039.7302783143161,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1020.6589616543491,1022.1576923175103,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,965.437822756589,999.5438403180763,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,912.2388336601439,950.3304269579979,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,841.1845129591121,873.1998397741838,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,794.8363424547509,852.1337214129775,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

arena_elo/results/latest/elo_results_t2i_generation.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e2e2b693cc44a7ba8b8abf1501f3f1ebda6c232b5ee79f760545705a7ca3248
-size 62419
+oid sha256:f923aeeaecb985cafb5ec70c59daef664f625b39bffc38a488e8066c701e1146
+size 62428

arena_elo/results/latest/t2i_generation_leaderboard.csv

@@ -1,10 +1,10 @@
 key,Model,Arena Elo rating (anony),Arena Elo rating (full),License,Organization,Link
-PlayGroundV2.5,PlayGroundV2.5,1226.9110139608943,1102.8780955386826,N/A,N/A,N/A
-StableCascade,StableCascade,1105.4281747413283,1059.3744882854992,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
-Playground v2,Playground v2,1090.337952005879,1100.0232585639621,Playground v2 Community License,Playground,https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic
-SDXLLightning,SDXLLightning,1042.9663858069512,1039.7571032040712,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
-PixArtAlpha,PixArtAlpha,1020.6589616543491,1022.4559565308926,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
-SDXL,SDXL,965.437822756589,999.8413183312834,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
-SDXLTurbo,SDXLTurbo,912.2388336601439,950.6036749825616,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
-OpenJourney,OpenJourney,841.1845129591121,873.4879879328548,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
-LCM,LCM,794.8363424547509,852.4223549305145,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7
+PlayGround V2.5,PlayGround V2.5,1226.9110139608943,1105.1039810421305,N/A,N/A,N/A
+StableCascade,StableCascade,1105.4281747413283,1059.2855612093028,stable-cascade-nc-community (other),Stability AI,https://huggingface.co/stabilityai/stable-cascade
+PlayGround V2,PlayGround V2,1090.337952005879,1099.3544589205903,N/A,N/A,N/A
+SDXLLightning,SDXLLightning,1042.9663858069512,1039.7302783143161,openrail++,ByteDance,https://huggingface.co/ByteDance/SDXL-Lightning
+PixArtAlpha,PixArtAlpha,1020.6589616543491,1022.1576923175103,openrail++,PixArt-alpha,https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS
+SDXL,SDXL,965.437822756589,999.5438403180763,openrail++,Stability AI,https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0
+SDXLTurbo,SDXLTurbo,912.2388336601439,950.3304269579979,sai-nc-community (other),Stability AI,https://huggingface.co/stabilityai/sdxl-turbo
+OpenJourney,OpenJourney,841.1845129591121,873.1998397741838,creativeml-openrail-m,PromptHero,https://huggingface.co/prompthero/openjourney
+LCM,LCM,794.8363424547509,852.1337214129775,MIT License,Tsinghua University,https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7

serve/vote_utils.py

@@ -3,6 +3,7 @@ import time
 import json
 import uuid
 import gradio as gr
+import regex as re
 from pathlib import Path
 from .utils import *
 from .log_utils import build_logger
@@ -344,6 +345,9 @@ def generate_igm(gen_func, state0, state1, text, model_name0, model_name1, reque
     ip = get_ip(request)
     igm_logger.info(f"generate. ip: {ip}")
     start_tstamp = time.time()
+    # Remove ### Model (A|B): from model name
+    model_name0 = re.sub(r"### Model A: ", "", model_name0)
+    model_name1 = re.sub(r"### Model B: ", "", model_name1)
     generated_image0, generated_image1 = gen_func(text, model_name0, model_name1)
     state0.prompt = text
     state1.prompt = text
@@ -400,6 +404,8 @@ def generate_igm_annoy(gen_func, state0, state1, text, model_name0, model_name1,
     ip = get_ip(request)
     igm_logger.info(f"generate. ip: {ip}")
     start_tstamp = time.time()
+    model_name0 = re.sub(r"### Model A: ", "", model_name0)
+    model_name1 = re.sub(r"### Model B: ", "", model_name1)
     generated_image0, generated_image1, model_name0, model_name1 = gen_func(text, model_name0, model_name1)
     state0.prompt = text
     state1.prompt = text
@@ -521,6 +527,8 @@ def generate_iem(gen_func, state0, state1, source_text, target_text, instruct_te
     ip = get_ip(request)
     igm_logger.info(f"generate. ip: {ip}")
     start_tstamp = time.time()
+    model_name0 = re.sub(r"### Model A: ", "", model_name0)
+    model_name1 = re.sub(r"### Model B: ", "", model_name1)
     generated_image0, generated_image1 = gen_func(source_text, target_text, instruct_text, source_image, model_name0, model_name1)
     state0.source_prompt = source_text
     state0.target_prompt = target_text
@@ -594,6 +602,8 @@ def generate_iem_annoy(gen_func, state0, state1, source_text, target_text, instr
     ip = get_ip(request)
     igm_logger.info(f"generate. ip: {ip}")
     start_tstamp = time.time()
+    model_name0 = re.sub(r"### Model A: ", "", model_name0)
+    model_name1 = re.sub(r"### Model B: ", "", model_name1)
     generated_image0, generated_image1, model_name0, model_name1 = gen_func(source_text, target_text, instruct_text, source_image, model_name0, model_name1)
     state0.source_prompt = source_text
     state0.target_prompt = target_text