Final Sotopia Task Submission

sotopia_space/benchmark.py

@@ -1,7 +1,7 @@
 import gradio as gr # type: ignore
 import pandas as pd
 from sotopia_space.constants import MODEL_OPTIONS
-from sotopia_space.utils import apply_length_penalty 
+from sotopia_space.utils import post_processing
 
 LP_MODE = "v2"
 original_df, ablation_df = None, None
@@ -16,13 +16,15 @@ def benchmark_table():
                 
     with gr.TabItem("Vs GPT-3.5", elem_id="od-benchmark-tab-table-ablation", id=0, elem_classes="subtab"):
         original_df = pd.read_json('data_dir/models_vs_gpt35.jsonl', lines=True)
-        default_main_df = apply_length_penalty(original_df, ablation_df, length_penalty=DEFAULT_LP, mode=LP_MODE, LP_original_dfs=LP_original_dfs) 
+        default_main_df = original_df
         default_main_df = default_main_df.sort_values(by="GOAL [0, 10]", ascending=False)
+        default_main_df = post_processing(default_main_df, None)
         # add a Rank column to the first columnn (starting from 1)
         default_main_df.insert(0, "Rank", range(1, 1 + len(default_main_df)))
+        
         with gr.Row():
             with gr.Column(scale=4):
-                gr.Markdown("**Vs GPT3.5**: The interlocutors are compared against GPT-3.5, the baseline model.")
+                gr.Markdown("<h3>**Vs GPT3.5**: The interlocutors are compared against GPT-3.5, the baseline model.")
         TYPES = ["number", "markdown", "number"]
         leaderboard_table = gr.components.Dataframe(
             value=default_main_df,
@@ -33,4 +35,4 @@ def benchmark_table():
             interactive=False,
             visible=True,
             min_width=60,
-            )
+            )

sotopia_space/constants.py

@@ -12,3 +12,28 @@ MODEL_OPTIONS = [
     # "together_ai/togethercomputer/llama-2-7b-chat",
     # "together_ai/togethercomputer/falcon-7b-instruct",
 ]
+
+MODEL_INFO = {
+    "Llama-2-13b-chat-hf.nosp": {"pretty_name": "Llama-2-13B-chat", "hf_model_id": "meta-llama/Llama-2-13b-chat-hf"},
+    "Llama-2-70b-chat-hf.nosp": {"pretty_name": "Llama-2-70B-chat", "hf_model_id": "meta-llama/Llama-2-70b-chat-hf"},
+    "Llama-2-7b-chat-hf.nosp": {"pretty_name": "Llama-2-7B-chat", "hf_model_id": "meta-llama/Llama-2-7b-chat-hf"},
+    "Llama-2-7b-chat-hf": {"pretty_name": "Llama-2-7B-chat (+sys prmpt)", "hf_model_id": "meta-llama/Llama-2-7b-chat-hf"},
+    "Mistral-7B-Instruct-v0.1": {"pretty_name": "Mistral-7B-Instruct", "hf_model_id": "mistralai/Mistral-7B-Instruct-v0.1"},
+    "Mistral-7B-Instruct-v0.2": {"pretty_name": "Mistral-7B-Instruct (v0.2)", "hf_model_id": "mistralai/Mistral-7B-Instruct-v0.2"},
+    "Mixtral-8x7B-Instruct-v0.1": {"pretty_name": "Mixtral-8x7B-Instruct", "hf_model_id": "mistralai/Mixtral-8x7B-Instruct-v0.1"},
+    "Nous-Hermes-2-Mixtral-8x7B-DPO": {"pretty_name": "Nous-Hermes-2-Mixtral-8x7B-DPO", "hf_model_id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO"},
+    "Yi-34B-Chat": {"pretty_name": "Yi-34B-Chat", "hf_model_id": "01-ai/Yi-34B"},
+    "gemini-1.0-pro": {"pretty_name": "gemini-1.0-pro", "hf_model_id": "https://blog.google/technology/ai/google-gemini-ai/"},
+    "gemma-7b-it": {"pretty_name": "Gemma-7B-it", "hf_model_id": "google/gemma-7b"},
+    "gpt-3.5-turbo-0125": {"pretty_name": "gpt-3.5-turbo-0125", "hf_model_id": "https://platform.openai.com/"},
+    "gpt-4-0125-preview": {"pretty_name": "gpt-4-0125-preview", "hf_model_id": "https://platform.openai.com/"},
+    "tulu-2-dpo-70b": {"pretty_name": "Tulu-2-dpo-70b", "hf_model_id": "cmu-lti/tulu-2-dpo-70b"},
+    "vicuna-13b-v1.5": {"pretty_name": "Vicuna-13b-v1.5", "hf_model_id": "lmsys/vicuna-13b-v1.5"},
+    "zephyr-7b-beta": {"pretty_name": "Zephyr-7b-beta", "hf_model_id": "HuggingFaceH4/zephyr-7b-beta"},
+    "mistral-large-2402": {"pretty_name": "Mistral-Large", "hf_model_id": "https://mistral.ai/news/mistral-large/"},
+    "claude-3-opus-20240229": {"pretty_name": "Claude 3 Opus", "hf_model_id": "https://www.anthropic.com/claude"},
+    "claude-3-sonnet-20240229": {"pretty_name": "Claude 3 Sonnet", "hf_model_id": "https://www.anthropic.com/claude"},
+    "zephyr-7b-gemma-v0.1": {"pretty_name": "Zephyr-7b-Gemma", "hf_model_id": "HuggingFaceH4/zephyr-7b-gemma-v0.1"},
+    "Starling-LM-7B-beta": {"pretty_name": "StarlingLM-7B-beta", "hf_model_id": "Nexusflow/Starling-LM-7B-beta"},
+    "dbrx-instruct": {"pretty_name": "DBRX Instruct", "hf_model_id": "databricks/dbrx-instruct"}
+}

sotopia_space/utils.py

@@ -1,3 +1,39 @@
+from ui_constants import column_names
+from sotopia_space.constants import MODEL_INFO
 
-def apply_length_penalty(original_df, ablation_df, length_penalty=0.2, mode='v1', LP_original_dfs=None):
-    return original_df 
+
+def make_clickable_model(model_name):
+    global MODEL_INFO
+    if model_name in MODEL_INFO:
+        if MODEL_INFO[model_name]["hf_model_id"].startswith("http"):
+            link = MODEL_INFO[model_name]["hf_model_id"]
+            return f'🔒 <a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{MODEL_INFO[model_name]["pretty_name"]}</a>'
+        else:
+            link = f"https://huggingface.co/{MODEL_INFO[model_name]['hf_model_id']}"
+            return f'🔥 <a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{MODEL_INFO[model_name]["pretty_name"]}</a>'
+    else:
+        return model_name
+    
+# Formats the columns
+def formatter(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x
+
+def post_processing(df, model_len_info):
+    if model_len_info:
+        df["Length"] = df["model_name"].apply(lambda x: model_len_info[x]["avg_len"])
+
+    for col in df.columns:
+        if col == "model_name":
+            df[col] = df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+        else:
+            df[col] = df[col].apply(formatter) # For numerical values 
+    df.rename(columns=column_names, inplace=True)
+    df.sort_values(by="GOAL [0, 10]", inplace=True, ascending=False)
+    # put the "Overall Elo" and "Task-Avg Elo" column to the front
+    # add the length info
+    df = df[["model_name", "GOAL [0, 10]"] + [col for col in df.columns if col not in ["model_name", "GOAL [0, 10]"]]]
+    return df