added automatic update of the best LLM models

app.py

@@ -27,6 +27,7 @@ from src.display_models.utils import (
     styled_message,
     styled_warning,
 )
+from src.manage_collections import update_collections
 from src.load_from_hub import get_all_requested_models, get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub
 from src.rate_limiting import user_submission_permission
 
@@ -88,6 +89,7 @@ snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="
 requested_models, users_to_submission_dates = get_all_requested_models(EVAL_REQUESTS_PATH)
 
 original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+update_collections(original_df.copy())
 leaderboard_df = original_df.copy()
 
 models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
@@ -306,7 +308,7 @@ def filter_models(
     if show_deleted:
         filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]

requirements.txt

@@ -25,7 +25,7 @@ gradio-client==0.5.0
 h11==0.14.0
 httpcore==0.17.0
 httpx==0.24.0
-huggingface-hub==0.16.4
+huggingface-hub==0.18.0
 idna==3.4
 Jinja2==3.1.2
 jsonschema==4.17.3

src/manage_collections.py

@@ -0,0 +1,65 @@
+import os
+import pandas as pd
+from pandas import DataFrame
+from huggingface_hub import get_collection, add_collection_item, delete_collection_item
+from huggingface_hub.utils._errors import HfHubHTTPError
+
+from src.display_models.model_metadata_type import ModelType
+from src.display_models.utils import AutoEvalColumn
+
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+path_to_collection = "HuggingFaceH4/current-best-models-of-the-open-llm-leaderboard-652d64cf619fc62beef5c2a3"
+intervals = {
+    "1B": pd.Interval(0, 1.5, closed="right"),
+    "3B": pd.Interval(2.5, 3.5, closed="neither"),
+    "7B": pd.Interval(6, 8, closed="neither"),
+    "13B": pd.Interval(10, 14, closed="neither"),
+    "30B":pd.Interval(25, 35, closed="neither"), 
+    "60B": pd.Interval(55, 65, closed="neither"),
+}
+
+def update_collections(df: DataFrame):
+    """This function updates the Open LLM Leaderboard model collection with the latest best models for 
+    each size category and type.
+    """
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+
+    cur_best_models = []
+
+    for type in ModelType:
+        if type.value.name == "": continue
+        for size in intervals:
+            # We filter the df to gather the relevant models
+            type_emoji = [t[0] for t in type.value.symbol]
+            filtered_df = df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+            numeric_interval = pd.IntervalIndex([intervals[size]])
+            mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+            filtered_df = filtered_df.loc[mask]
+
+            best_models = list(filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name])
+
+            # We add them one by one to the leaderboard
+            for model in best_models:
+                # We can use collection = get_collection to grab the id of the last item, then place it where we want using update_collection but it's costly...
+                # We could also remove exists_ok to update the note to include the date of apparition of the model for ex.
+                try:
+                    add_collection_item(
+                        path_to_collection, 
+                        item_id=model, 
+                        item_type="model", 
+                        exists_ok=True, 
+                        note=f"Best {type.to_str(' ')} model of {size} on the leaderboard today!", 
+                        token=H4_TOKEN
+                    )
+                    cur_best_models.append(model)
+                    break
+                except HfHubHTTPError:
+                    continue
+
+    collection = get_collection(path_to_collection, token=H4_TOKEN)
+    for item in collection.items:
+        if item.item_id not in cur_best_models:
+            delete_collection_item(collection_slug=path_to_collection, item_object_id=item.item_object_id, token=H4_TOKEN)
+