Upload 3 files

src/leaderboard_utils.py

@@ -29,56 +29,42 @@ def get_github_data():
     json_data = response.json()
     versions = json_data['versions']
 
-    # Sort version names - latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
-        key=lambda v: float(v[1:]),
+        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  # {{ edit_1 }}: Corrected slicing to handle 'v' prefix
         reverse=True
-    )
-    print(f"Found {len(version_names)} versions from get_github_data(): {version_names}.")
+    )   
 
     # Get Last updated date of the latest version
     latest_version = version_names[0]
     latest_date = next(
         ver['date'] for ver in versions if ver['version'] == latest_version
     )
-    formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
+    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")  # {{ edit_1 }}: Updated date format
 
     # Get Leaderboard data - for text-only + multimodal
     github_data = {}
 
-    # Collect Dataframes
-    text_dfs = []
     mm_dfs = []
-
+    mm_date = ""
+    mm_flag = True
     for version in version_names:
-        # Collect CSV data in descending order of clembench-runs versions
-        # Collect Text-only data
-        text_url = f"{base_repo}{version}/results.csv"
-        csv_response = requests.get(text_url)
-        if csv_response.status_code == 200:
-            df = pd.read_csv(StringIO(csv_response.text))
+        # Check if version ends with 'multimodal' before constructing the URL
+        mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
+        mm_url = f"{base_repo}{version}{mm_suffix}/results.csv"  # {{ edit_1 }}: Conditional suffix for multimodal
+        mm_response = requests.get(mm_url)
+        if mm_response.status_code == 200:
+            df = pd.read_csv(StringIO(mm_response.text))
             df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-            text_dfs.append(df)
-        else:
-            print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
-
-        # Collect Multimodal data
-        if float(version[1:]) >= 1.6:
-            mm_url = f"{base_repo}{version}_multimodal/results.csv"
-            mm_response = requests.get(mm_url)
-            if mm_response.status_code == 200:
-                df = pd.read_csv(StringIO(mm_response.text))
-                df = process_df(df)
-                df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
-                mm_dfs.append(df)
-        else:
-            print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {csv_response.status_code}. Please ignore this message if multimodal results are not available for this version")
-
-    github_data["text"] = text_dfs
+            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
+            mm_dfs.append(df)
+            if mm_flag:
+                mm_date = next(ver['date'] for ver in versions if ver['version'] == version)
+                mm_date = datetime.strptime(mm_date, "%Y-%m-%d").strftime("%d %b %Y")
+                mm_flag = False
+
     github_data["multimodal"] = mm_dfs
-    github_data["date"] = formatted_date
+    github_data["date"] = mm_date
 
     return github_data
 
@@ -136,4 +122,4 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
     # Filter dataframe based on queries in 'Model' column
     filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
 
-    return filtered_df
+    return filtered_df

src/plot_utils.py

@@ -127,27 +127,22 @@ def split_models(model_list: list):
     """
     Split the models into open source and commercial
     """
-
     open_models = []
     commercial_models = []
-    open_backends = {"huggingface_local", "huggingface_multimodal", "openai_compatible"}  # Define backends considered as open
-
+    
     # Load model registry data from main repo
     model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
     response = requests.get(model_registry_url)
 
     if response.status_code == 200:
         json_data = json.loads(response.text)
-        # Classify as Open or Commercial based on the defined backend in the model registry
-        backend_mapping = {}
 
         for model_name in model_list:
-            model_prefix = model_name.split('-')[0]  # Get the prefix part of the model name
             for entry in json_data:
-                if entry["model_name"].startswith(model_prefix):
-                    backend = entry["backend"]
-                    # Classify based on backend
-                    if backend in open_backends:
+                if entry["model_name"] == model_name:
+                    open_model = entry["open_weight"]
+                    
+                    if open_model:
                         open_models.append(model_name)
                     else:
                         commercial_models.append(model_name)
@@ -178,7 +173,7 @@ def update_open_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Open Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["multimodal"][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -198,7 +193,7 @@ def update_closed_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Closed Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["multimodal"][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -217,7 +212,7 @@ def get_plot_df(leaderboard: str = TEXT_NAME) -> pd.DataFrame:
         DataFrame with model data.
     """
     github_data = get_github_data()
-    return github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    return github_data["multimodal"][0]
 
 
 """

src/version_utils.py

@@ -31,41 +31,30 @@ def get_versions_data():
     json_data = response.json()
     versions = json_data['versions']
 
-    # Sort version names - latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
-        key=lambda v: float(v[1:]),
+        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  # {{ edit_1 }}: Corrected slicing to handle 'v' prefix
         reverse=True
-    )
-    print(f"Found {len(version_names)} versions from get_versions_data(): {version_names}.")
+    )   
 
     # Get Last updated date of the latest version
     latest_version = version_names[0]
     latest_date = next(
         ver['date'] for ver in versions if ver['version'] == latest_version
     )
-    formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
+    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")  # {{ edit_1 }}: Updated date format
 
     # Get Versions data
     versions_data = {"latest": latest_version, "date": formatted_date}
 
-    # Collect Dataframes
-    dfs = []
 
     for version in version_names:
-        text_url = f"{base_repo}{version}/results.csv"
-        mm_url = f"{base_repo}{version}_multimodal/results.csv"
-        quant_url = f"{base_repo}{version}_quantized/results.csv"
-
-        # Text Data
-        response = requests.get(text_url)
-        if response.status_code == 200:
-            df = pd.read_csv(StringIO(response.text))
-            df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version] = df
+        if version.endswith("multimodal"):
+            version_suffix = ""
         else:
-            print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {response.status_code}")
+            version_suffix = "_multimodal"
+
+        mm_url = f"{base_repo}{version}{version_suffix}/results.csv"
 
         # Multimodal Data
         mm_response = requests.get(mm_url)
@@ -73,20 +62,10 @@ def get_versions_data():
             mm_df = pd.read_csv(StringIO(mm_response.text))
             mm_df = process_df(mm_df)
             mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version+"_multimodal"] = mm_df
+            versions_data[version+version_suffix] = mm_df
         else:
             print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if multimodal results are not available for this version")
 
-        # Multimodal Data
-        q_response = requests.get(quant_url)
-        if q_response.status_code == 200:
-            q_df = pd.read_csv(StringIO(q_response.text))
-            q_df = process_df(q_df)
-            q_df = q_df.sort_values(by=q_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version + "_quantized"] = q_df
-        else:
-            print(f"Failed to read quantized leaderboard CSV file for version: {version}: Status Code: {mm_response.status_code}. Please ignore this message if quantized results are not available for this version")
-
     return versions_data