Move eval output files to separate repo

app.py

@@ -2,29 +2,36 @@ import os
 import streamlit as st
 import json
 import tarfile
+from huggingface_hub import HfFileSystem
 
+hf_fs = HfFileSystem(token=os.getenv("HF_TOKEN"))
 st.set_page_config(layout="wide")
 
 # Disable scroll bar
 st.html("<style> .main {overflow: hidden} </style>")
 
-PARENT_DIR: str = os.path.join(os.path.dirname(os.path.abspath(__file__)))
-EVAL_DIR: str = os.path.join(PARENT_DIR, "eval-results")
+DATASET_ID: str = "LLM360/k2-eval-gallery"
+EVAL_DIR: str = os.path.join("datasets", DATASET_ID, "k2-eval-results")
 st.title("K2 Evaluation Gallery")
 st.markdown("""The K2 gallery allows one to browse the output of various evaluations on intermediate K2 checkpoints, which provides an intuitive understanding on how the model develops and improves over time.""")
 
 
+def hf_listdir(parent_dir: str):
+    return (os.path.basename(file) for file in hf_fs.ls(
+        parent_dir, detail=False
+    ))
+
+
 with st.sidebar:
     html = f"<img src='https://www.llm360.ai/images/logo-highres.png' width='100' /><img src='https://huggingface.co/spaces/LLM360/k2-eval-gallery/raw/main/k2-logo.svg' width='100' />"
     st.markdown(html, unsafe_allow_html=True)
 
     metric = st.radio(
-        "Choose a metric", options=os.listdir(os.path.join(EVAL_DIR)),
-        help="type of evaluation benchmark task"
+        "Choose a metric", options=hf_listdir(EVAL_DIR), help="type of evaluation benchmark task"
     )
 
     n_shot = st.radio(
-        "Selece an n-shot number", os.listdir(os.path.join(EVAL_DIR, metric)),
+        "Selece an n-shot number", hf_listdir(os.path.join(EVAL_DIR, metric)),
         help="number of examples included in few-shot prompting"
     )
 
@@ -32,16 +39,18 @@ col1, col2 = st.columns(2)
 
 def render_column(col_label):
     st.header(f"Checkpoint {col_label}")
-    ckpt = st.select_slider('Select a checkpoint', sorted(os.listdir(os.path.join(EVAL_DIR, metric, n_shot))), key=col_label + '1', help="checkpoint index from 3 to 360")
+    ckpt = st.select_slider('Select a checkpoint', sorted(hf_listdir(os.path.join(EVAL_DIR, metric, n_shot))), key=col_label + '1', help="checkpoint index from 3 to 360")
     st.write(f'Veiwing Evaluation Results for Checkpoint: `{ckpt}`')
     suffix, result_file = ".tar.gz", "results.json"
-    file_list: list = sorted(f_name[:-len(suffix)] for f_name in os.listdir(os.path.join(EVAL_DIR, metric, n_shot, ckpt)))
+    file_list: list = sorted(f_name[:-len(suffix)] for f_name in hf_listdir(os.path.join(EVAL_DIR, metric, n_shot, ckpt)))
     if result_file in file_list:
         file_list.remove(result_file)
         file_list = file_list + [result_file]
     file = st.selectbox("Select a file", file_list, key=col_label + '2', help="a list of raw output files from evaluation results")
     file += suffix
-    with tarfile.open(os.path.join(EVAL_DIR, metric, n_shot, ckpt, file), "r:gz") as tar:
+    with tarfile.open(fileobj=hf_fs.open(
+        os.path.join(EVAL_DIR, metric, n_shot, ckpt, file), "rb"
+    ), mode="r:gz") as tar:
         f = tar.extractfile(tar.next())
         eval_json = json.load(f)
         if isinstance(eval_json, list):

eval-results/arc_challenge/25/ckpt_003/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:09427034954ac308622fa4f7a9150905d6176d7e836163af90405178d46f4b1b
-size 2267278

eval-results/arc_challenge/25/ckpt_003/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2cbc83b74607ff9ae9840a8900cf9e84833b901f62cecafe2c360df8c963ebd2
-size 971

eval-results/arc_challenge/25/ckpt_006/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:95c30af4e510049c3caa20681f0d6eb6d0323d3085535174a990a831ea2e11b4
-size 2267835

eval-results/arc_challenge/25/ckpt_006/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d90cfa213c35d375c1ecfaceff0975cc6ed9c33da3e97c3be61cf59b1ac7bce0
-size 975

eval-results/arc_challenge/25/ckpt_009/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e12e3e33e6233fc6d25d6199e7ccfdee658772eef6f9df518f2d838943880cb1
-size 2268278

eval-results/arc_challenge/25/ckpt_009/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:65b5ce9843ae6b35208561da2f6733cacd7a15fae9d11587e726581b63461475
-size 974

eval-results/arc_challenge/25/ckpt_012/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f960c892a55eafa725e29e4b383aff11119d3380ddfed868684f832f7d7645ed
-size 2268579

eval-results/arc_challenge/25/ckpt_012/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2e74ad787c6ee590c99a0051d1fa2e3b857ea3709239e4c3cf4a0f3895ca86e9
-size 970

eval-results/arc_challenge/25/ckpt_015/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:980f5c264ba7a4775d8978b9e987fcdf2d810b1adb48e79b777d1bbeca05eb19
-size 2268914

eval-results/arc_challenge/25/ckpt_015/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:80d059415d0961d205523b11a7f8017d6a10a08775a1ee3c7fe01e97bef3d7d1
-size 974

eval-results/arc_challenge/25/ckpt_018/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8c430d1d41fcdb8600ece38265a37e877759ed56b0af89042428572563bc63ca
-size 2269072

eval-results/arc_challenge/25/ckpt_018/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:baaf4fabdb81b35b3aa69c18192eeffb29fc831ccac0045be40c3aedaf772112
-size 974

eval-results/arc_challenge/25/ckpt_021/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b41e2595fdbd6dd8ec0c4c7b2a5ac23afb8543c05c6b19ed5d5d2ddf614f5ffa
-size 2268834

eval-results/arc_challenge/25/ckpt_021/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:830d2a5b9ffdcce4119ae5f7ed666e6c9a426e4b2b29a33cf935b40bd279fe85
-size 972

eval-results/arc_challenge/25/ckpt_024/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f1f9a4714f1855c5349bc378e34dbbd4002aa8d13673ab85f791a142a0bfff34
-size 2268988

eval-results/arc_challenge/25/ckpt_024/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:662aef6e1f84f6308d000911cff5f9854fef44d0bdcc7a8a95e025f0f1ef7098
-size 974

eval-results/arc_challenge/25/ckpt_027/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5980e929f0e2c2463b0a7bd7e0d82bc72371fd15cd90bc7757c1448bfc469668
-size 2268923

eval-results/arc_challenge/25/ckpt_027/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:90991d83b81e83663047204190854214ab244ca72d74b02ba7a5adfd9c1b291b
-size 972

eval-results/arc_challenge/25/ckpt_030/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b19466827850547ee1035c5ba148843a2ee808366f4c0b6102a9d79f3424952d
-size 2269233

eval-results/arc_challenge/25/ckpt_030/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a63fe0c13dcd0bc5c1b8b10e251289165cf9a69ecc2d9cc2cc68c59ad610bc74
-size 973

eval-results/arc_challenge/25/ckpt_033/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8b1a2eb36c241fe2f486c6e1dd5ba800e704f6d296e46d4f86295a672aa5624c
-size 2269431

eval-results/arc_challenge/25/ckpt_033/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dc5875fb872f435db3c353db8d888f7af5c75363ecd3220ec014b081367eb4fb
-size 974

eval-results/arc_challenge/25/ckpt_036/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b841343cf5930896d3cbb976a81ed859258d0c8510cbab4bd848bcdf9407c5f
-size 2269380

eval-results/arc_challenge/25/ckpt_036/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4111776450ae3c2807c545f6da974a45871db6570aafe1074ef23508a59ddad5
-size 972

eval-results/arc_challenge/25/ckpt_039/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:372bb01964b106934e6c9fd7eff34b59b06ee9cf7d27da775f82ec0f61c14826
-size 2268990

eval-results/arc_challenge/25/ckpt_039/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a07220830af85533ec34100ab28e2d6985094dd9f1631d435160c5609abd8ea0
-size 975

eval-results/arc_challenge/25/ckpt_042/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9768098f0696990e80e0b58ecc3e8bca1f56341ba862b541c20235e7e88054f7
-size 2269356

eval-results/arc_challenge/25/ckpt_042/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5df47aecf21d09d02d0ec953dbb799cf3bd51142d351361e7a3992a8a6a00525
-size 968

eval-results/arc_challenge/25/ckpt_045/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:46ae8a45fe3e5d87305192c0630cc55f19e81cea981afd65c680f99c7059ac5b
-size 2269312

eval-results/arc_challenge/25/ckpt_045/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c79a94693a01df16f6b6455af64d0b4768cd354c12ae42cec9e16911507aaf47
-size 973

eval-results/arc_challenge/25/ckpt_048/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ab961c0ad4106f1acd4b2d133b13a9b07cdc8f9a19f49a1c2b6e928dd393492a
-size 2269394

eval-results/arc_challenge/25/ckpt_048/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eac79879081707f3624fba1c2db84e9d8ff2c6c968b7790ee7c03fcfab42e917
-size 973

eval-results/arc_challenge/25/ckpt_051/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4d2da45810d92eec9f94e967107fcf5fa9bf2c9459d95781a6e7e837db8082bb
-size 2269532

eval-results/arc_challenge/25/ckpt_051/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd41bf19e04056050a487957ada2a118574fb890a5d39324588f44895086d712
-size 971

eval-results/arc_challenge/25/ckpt_054/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef3ecf947554cee711f65998be35ef4b57a175f12af949fe126babf7ae7187dd
-size 2269331

eval-results/arc_challenge/25/ckpt_054/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9ccc6c4fb026aaa6b174c59ffc94a7d4f44bb9fbf4beba51ce27fd1faf45a1e9
-size 973

eval-results/arc_challenge/25/ckpt_057/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1c97349dd93853dd0c9e9db347dad323e505da463be35575056e43060b8c1819
-size 2269430

eval-results/arc_challenge/25/ckpt_057/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bc0aac4e3bb5d891a318eed5cbe953df65a43104826fcc3e03c7d168e5d40b30
-size 964

eval-results/arc_challenge/25/ckpt_060/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bf5ee6c91a7903060f2723655373ae962e686d76f5d831a8dc4ce655e6c651ac
-size 2269532

eval-results/arc_challenge/25/ckpt_060/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:be8dca4eec8608eb4f575807b1f5615dfbbaadc0112096d0766f0b8925b82612
-size 975

eval-results/arc_challenge/25/ckpt_063/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:14dc09d0ab66e6af7f72d13d6b10e446005696908c6ef9a4a57870ab231faa04
-size 2269441

eval-results/arc_challenge/25/ckpt_063/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ae5f5d9f014590aeb3dd9413ec006f22bb60d90e81b202712501bce835d28663
-size 975

eval-results/arc_challenge/25/ckpt_066/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c27788dd79df3ec9ecd326e45caa66e3a160dd07a055592923a5a1d19134ffe7
-size 2269529

eval-results/arc_challenge/25/ckpt_066/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b5c4c3b1af009b285e38523923ea255a7641dc5b08b33f96cfa2f9f039f91463
-size 972

eval-results/arc_challenge/25/ckpt_069/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e86a0f0246bbcc87f8f075662eabaf32ba4681454dd1abcdd47b02284bf7933c
-size 2269509

eval-results/arc_challenge/25/ckpt_069/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e8acb9ac6d6bac6d911fc7a58770591db68b6a299051d377f384ca386c72a610
-size 974

eval-results/arc_challenge/25/ckpt_072/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:711988ec65c8789b75873f0a5198d83307e0cf03c21b199ded3bfa75cb036059
-size 2269606

eval-results/arc_challenge/25/ckpt_072/results.json.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6a26d833ecbff817c18be5c2fe74d079f8dc6454051031a8c23de74ee0552075
-size 974

eval-results/arc_challenge/25/ckpt_075/arc_challenge.jsonl.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e1cfa259dde58cc3a99241e8b517866b2294aaad74e15a0c48e45975cf0b3dfe
-size 2269367