add data collection script

generate_overview_json.py

@@ -10,13 +10,16 @@ def get_num_parameters(model_name: str) -> int:
 
 
 def main():
-    results = {}
-    for pfin in Path(__file__).parent.joinpath("evals").rglob("*.json"):
+    evals_dir = Path(__file__).parent.joinpath("evals")
+    pf_overview = evals_dir.joinpath("models.json")
+    results = json.loads(pf_overview.read_text(encoding="utf-8")) if pf_overview.exists() else {}
+
+    for pfin in evals_dir.rglob("*.json"):
         if pfin.stem == "models":
             continue
         short_name = pfin.stem.split("_")[2]
-        if short_name not in results:
-            results[short_name] = {}
+        if short_name in results:
+            continue
 
         data = json.loads(pfin.read_text(encoding="utf-8"))
         if "config" not in data:
@@ -41,6 +44,7 @@ def main():
         results[short_name]["num_parameters"] = get_num_parameters(model_args["pretrained"])
 
     pprint(results)
+    pf_overview.write_text(json.dumps(results, indent=4), encoding="utf-8")
 
 
 if __name__ == '__main__':