get model type info from request file

app.py

@@ -328,7 +328,7 @@ with demo:
                     )
                     filter_columns = gr.Radio(
                         label="⏚ Filter model types",
-                        choices = ["all", "🟢 base", "🔶 instruction-tuned", "🟦 RL-tuned"],
+                        choices = ["all", "🟢 base", "🔶 fine-tuned", "🟦 RL-tuned"],
                         value="all",
                         elem_id="filter-columns"
                     )
@@ -404,7 +404,7 @@ with demo:
                         False, label="Private", visible=not IS_PUBLIC
                     )
                     model_type = gr.Dropdown(
-                        choices=["pretrained", "fine-tuned", "with RL"], 
+                        choices=["pretrained", "fine-tuned", "RL-tuned"], 
                         label="Model type", 
                         multiselect=False,
                         value=None,

src/auto_leaderboard/load_results.py

@@ -119,12 +119,6 @@ def get_eval_results(is_public) -> List[EvalResult]:
 
         up_to_date = files[-1]
 
-        if len(files) > 1:
-            print(root)
-            print(files)
-            print(up_to_date)
-            print("===")
-
         json_filepaths.append(os.path.join(root, up_to_date))
 
     eval_results = {}

src/auto_leaderboard/model_metadata_type.py

@@ -15,13 +15,13 @@ class ModelInfo:
 model_type_symbols = {
     "fine-tuned": "🔶",
     "pretrained": "🟢",
-    "with RL": "🟦",
+    "RL-tuned": "🟦",
 }
 
 class ModelType(Enum):
     PT = ModelInfo(name="pretrained", symbol="🟢")
-    SFT = ModelInfo(name="finetuned", symbol="🔶")
-    RL = ModelInfo(name="with RL", symbol="🟦")
+    SFT = ModelInfo(name="fine-tuned", symbol="🔶")
+    RL = ModelInfo(name="RL-tuned", symbol="🟦")
 
 
 TYPE_METADATA: Dict[str, ModelType] = {
@@ -532,21 +532,31 @@ TYPE_METADATA: Dict[str, ModelType] = {
 def get_model_type(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
         # Todo @clefourrier once requests are connected with results 
-        is_delta = False # (model_data["weight_type"] != "Original")
         # Stored information
         request_file = os.path.join("eval-queue", model_data["model_name_for_query"] + "_eval_request_*" + ".json")
         request_file = glob.glob(request_file)
 
+        if len(request_file) == 0:
+            model_data[AutoEvalColumn.model_type.name] = "Unknown"
+            model_data[AutoEvalColumn.model_type_symbol.name] = "No request file"
+            continue
+
+        request_file = request_file[0]
+
         try:
-            request_file = request_file[0]
             with open(request_file, "r") as f:
                 request = json.load(f)
-            model_type = request["model_type"]
             is_delta = request["weight_type"] != "Original"
+        except Exception:
+            is_delta = False
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            model_type = request["model_type"]
             model_data[AutoEvalColumn.model_type.name] = model_type
             model_data[AutoEvalColumn.model_type_symbol.name] = model_type_symbols[model_type] + ("🔺" if is_delta else "")
         except Exception:
-            if model_data["model_name_for_query"] in TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol + ("🔺" if is_delta else "")
+            model_data[AutoEvalColumn.model_type.name] = "Unknown"
+            model_data[AutoEvalColumn.model_type_symbol.name] = "Add type to request files"