Spaces:
Running
Running
| import json | |
| import logging | |
| import os | |
| import time | |
| import pandas as pd | |
| from huggingface_hub import snapshot_download | |
| from src.envs import DATA_PATH, HF_TOKEN_PRIVATE | |
| # Configure logging | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") | |
| def time_diff_wrapper(func): | |
| def wrapper(*args, **kwargs): | |
| start_time = time.time() | |
| result = func(*args, **kwargs) | |
| end_time = time.time() | |
| diff = end_time - start_time | |
| logging.info("Time taken for %s: %s seconds", func.__name__, diff) | |
| return result | |
| return wrapper | |
| def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5): | |
| """Download dataset with exponential backoff retries.""" | |
| os.makedirs(local_dir, exist_ok=True) | |
| attempt = 0 | |
| while attempt < max_attempts: | |
| try: | |
| logging.info("Downloading %s to %s", repo_id, local_dir) | |
| snapshot_download( | |
| repo_id=repo_id, | |
| local_dir=local_dir, | |
| cache_dir='./tmp', | |
| repo_type=repo_type, | |
| tqdm_class=None, | |
| token=HF_TOKEN_PRIVATE, | |
| etag_timeout=30, | |
| max_workers=8, | |
| local_dir_use_symlinks=False | |
| ) | |
| logging.info("Download successful") | |
| return | |
| except Exception as e: | |
| wait_time = backoff_factor**attempt | |
| logging.error("Error downloading %s: %s, retrying in %ss", repo_id, e, wait_time) | |
| time.sleep(wait_time) | |
| attempt += 1 | |
| logging.error("Failed to download %s after %s attempts", repo_id, max_attempts) | |
| def download_openbench(): | |
| # download prev autogenerated leaderboard files | |
| download_dataset("Vikhrmodels/s-shlepa-metainfo", DATA_PATH) | |
| # download answers of different models that we trust | |
| download_dataset("Vikhrmodels/s-openbench-eval", "m_data") | |
| def build_leadearboard_df(): | |
| # Retrieve the leaderboard DataFrame | |
| with open(f"{os.path.abspath(DATA_PATH)}/leaderboard.json", "r", encoding="utf-8") as eval_file: | |
| f=json.load(eval_file) | |
| print(f) | |
| df = pd.DataFrame.from_records(f) | |
| if 'mmluproru' in list(df.columns): | |
| df['mmluproru'] = df['mmluproru'].fillna(0) | |
| else: | |
| df['mmluproru'] = 0 | |
| leaderboard_df = df[['model','mmluproru','moviesmc','musicmc','lawmc','booksmc','model_dtype','ppl']] | |
| leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc','mmluproru']].mean(axis=1).values | |
| # print(leaderboard_df.columns) | |
| if len(leaderboard_df)>3: | |
| leaderboard_df = leaderboard_df[leaderboard_df['mmluproru']!=0] | |
| logging.info("Leaderboard DataFrame shape:", leaderboard_df) | |
| leaderboard_df.sort_values(by='avg',ascending=False,inplace=True,axis=0) | |
| numeric_cols = leaderboard_df.select_dtypes(include=['number']).columns | |
| # print(numeric_cols) | |
| leaderboard_df[numeric_cols] = leaderboard_df[numeric_cols].round(3) | |
| return leaderboard_df.copy() | |