Upload 7 files

src/leaderboard_utils.py

@@ -29,20 +29,18 @@ def get_github_data():
     json_data = response.json()
     versions = json_data['versions']
 
-    # Sort version names - latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
-        key=lambda v: float(v[1:]),
+        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  # {{ edit_1 }}: Corrected slicing to handle 'v' prefix
         reverse=True
-    )
-    print(f"Found {len(version_names)} versions from get_github_data(): {version_names}.")
+    )   
 
     # Get Last updated date of the latest version
     latest_version = version_names[0]
     latest_date = next(
         ver['date'] for ver in versions if ver['version'] == latest_version
     )
-    formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
+    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")  # {{ edit_1 }}: Updated date format
 
     # Get Leaderboard data - for text-only + multimodal
     github_data = {}
@@ -54,28 +52,27 @@ def get_github_data():
     for version in version_names:
         # Collect CSV data in descending order of clembench-runs versions
         # Collect Text-only data
-        text_url = f"{base_repo}{version}/results.csv"
-        csv_response = requests.get(text_url)
-        if csv_response.status_code == 200:
-            df = pd.read_csv(StringIO(csv_response.text))
-            df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-            text_dfs.append(df)
-        else:
-            print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
-
-        # Collect Multimodal data
-        if float(version[1:]) >= 1.6:
-            mm_url = f"{base_repo}{version}_multimodal/results.csv"
-            mm_response = requests.get(mm_url)
-            if mm_response.status_code == 200:
-                df = pd.read_csv(StringIO(mm_response.text))
+        if len(version.split('_')) == 1: 
+            text_url = f"{base_repo}{version}/results.csv"
+            csv_response = requests.get(text_url)
+            if csv_response.status_code == 200:
+                df = pd.read_csv(StringIO(csv_response.text))
                 df = process_df(df)
-                df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
-                mm_dfs.append(df)
-        else:
-            print(f"Failed to read multimodal leaderboard CSV file for version: {version}: Status Code: {csv_response.status_code}. Please ignore this message if multimodal results are not available for this version")
-
+                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
+                text_dfs.append(df)
+            else:
+                print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
+
+        # Check if version ends with 'multimodal' before constructing the URL
+        mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
+        mm_url = f"{base_repo}{version}{mm_suffix}/results.csv"  # {{ edit_1 }}: Conditional suffix for multimodal
+        mm_response = requests.get(mm_url)
+        if mm_response.status_code == 200:
+            df = pd.read_csv(StringIO(mm_response.text))
+            df = process_df(df)
+            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
+            mm_dfs.append(df)
+      
     github_data["text"] = text_dfs
     github_data["multimodal"] = mm_dfs
     github_data["date"] = formatted_date
@@ -137,3 +134,4 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
     filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
 
     return filtered_df
+

src/plot_utils.py

@@ -127,27 +127,22 @@ def split_models(model_list: list):
     """
     Split the models into open source and commercial
     """
-
     open_models = []
     commercial_models = []
-    open_backends = {"huggingface_local", "huggingface_multimodal", "openai_compatible"}  # Define backends considered as open
-
+    
     # Load model registry data from main repo
     model_registry_url = "https://raw.githubusercontent.com/clp-research/clembench/main/backends/model_registry.json"
     response = requests.get(model_registry_url)
 
     if response.status_code == 200:
         json_data = json.loads(response.text)
-        # Classify as Open or Commercial based on the defined backend in the model registry
-        backend_mapping = {}
 
         for model_name in model_list:
-            model_prefix = model_name.split('-')[0]  # Get the prefix part of the model name
             for entry in json_data:
-                if entry["model_name"].startswith(model_prefix):
-                    backend = entry["backend"]
-                    # Classify based on backend
-                    if backend in open_backends:
+                if entry["model_name"] == model_name:
+                    open_model = entry["open_weight"]
+                    
+                    if open_model:
                         open_models.append(model_name)
                     else:
                         commercial_models.append(model_name)

src/version_utils.py

@@ -31,20 +31,18 @@ def get_versions_data():
     json_data = response.json()
     versions = json_data['versions']
 
-    # Sort version names - latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
-        key=lambda v: float(v[1:]),
+        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  # {{ edit_1 }}: Corrected slicing to handle 'v' prefix
         reverse=True
-    )
-    print(f"Found {len(version_names)} versions from get_versions_data(): {version_names}.")
+    )   
 
     # Get Last updated date of the latest version
     latest_version = version_names[0]
     latest_date = next(
         ver['date'] for ver in versions if ver['version'] == latest_version
     )
-    formatted_date = datetime.strptime(latest_date, "%Y/%m/%d").strftime("%d %b %Y")
+    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y")  # {{ edit_1 }}: Updated date format
 
     # Get Versions data
     versions_data = {"latest": latest_version, "date": formatted_date}