update

beta-cli.py

@@ -44,7 +44,7 @@ eval_requests: list[EvalRequest] = get_eval_requests(job_status=current_finished
 # Sort the evals by priority (first submitted first run)
 eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
 
-eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
+eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
 
 result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
 result_name_to_result = {r.eval_name: r for r in eval_results}
@@ -64,3 +64,8 @@ for eval_request in eval_requests:
 
         if task_name not in eval_result.results:
             print('RUN THIS ONE!', result_name, task_name)
+
+raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
+all_data_json = [v.to_dict() for v in raw_data if v.is_complete()]
+
+breakpoint()

src/leaderboard/read_evals.py

@@ -123,6 +123,12 @@ class EvalResult:
         except Exception:
             print(f"Could not find request file for {self.org}/{self.model}")
 
+    def is_complete(self) -> bool:
+        for task in Tasks:
+            if task.value.benchmark not in self.results:
+                return False
+        return True
+
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)

src/populate.py

@@ -15,7 +15,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     # EvalResult(eval_name='EleutherAI_pythia-1.3b_torch.float16', full_model='EleutherAI/pythia-1.3b', org='EleutherAI', model='pythia-1.3b', revision='34b668ff0acfe56f2d541aa46b385557ee39eb3f', results={'arc:challenge': 31.14334470989761, 'hellaswag': 51.43397729535949, 'hendrycksTest': 26.55151159544371, 'truthfulqa:mc': 39.24322830092449, 'winogrande': 57.37963693764798, 'gsm8k': 0.9855951478392722, 'drop': 4.056312919463095}, precision='torch.float16', model_type=<ModelType.PT: ModelTypeDetails(name='pretrained', symbol='🟢')>, weight_type='Original', architecture='GPTNeoXForCausalLM', license='apache-2.0', likes=7, num_params=1.312, date='2023-09-09T10:52:17Z', still_on_hub=True)
     # EvalResult and get_raw_eval_results are defined in ./src/leaderboard/read_evals.py, the results slots are not hardcoded
     raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    all_data_json = [v.to_dict() for v in raw_data if v.is_complete()]
     # all_data_json.append(baseline_row)
     filter_models(all_data_json)