column fix

app.py

@@ -27,6 +27,8 @@ GPT_4_EVAL_REPO = "HuggingFaceH4/open_llm_leaderboard_oai_evals"
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 ADD_PLOTS = False
 
+EVAL_REQUESTS_PATH = "auto_evals/eval_requests"
+
 api = HfApi()
 
 
@@ -37,10 +39,10 @@ def restart_space():
 
 auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models = load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
 
-COLS = [c.name for c in fields(AutoEvalColumn)]
-TYPES = [c.type for c in fields(AutoEvalColumn)]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default]
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 if not IS_PUBLIC:
     COLS.insert(2, AutoEvalColumn.is_8bit.name)
@@ -95,14 +97,14 @@ def get_evaluation_queue_df():
 
     entries = [
         entry
-        for entry in os.listdir("auto_evals/eval_requests")
+        for entry in os.listdir(EVAL_REQUESTS_PATH)
         if not entry.startswith(".")
     ]
     all_evals = []
 
     for entry in entries:
         if ".json" in entry:
-            file_path = os.path.join("auto_evals/eval_requests", entry)
+            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
             with open(file_path) as fp:
                 data = json.load(fp)
 
@@ -115,11 +117,11 @@ def get_evaluation_queue_df():
             # this is a folder
             sub_entries = [
                 e
-                for e in os.listdir(f"auto_evals/eval_requests/{entry}")
+                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
                 if not e.startswith(".")
             ]
             for sub_entry in sub_entries:
-                file_path = os.path.join("auto_evals/eval_requests", entry, sub_entry)
+                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
 
@@ -244,7 +246,7 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    OUT_DIR = f"auto_evals/eval_requests/{user_name}"
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
 

src/assets/text_content.py

@@ -1,4 +1,8 @@
 CHANGELOG_TEXT = f"""
+## [2023-06-19]
+- Added model type column
+- Hid revision and 8bit columns since all models are the same atm
+
 ## [2023-06-16]
 - Refactored code base
 - Added new columns: number of parameters, hub likes, license

src/auto_leaderboard/get_model_metadata.py

@@ -2,6 +2,7 @@ import re
 from typing import List
 
 from src.utils_display import AutoEvalColumn
+from src.auto_leaderboard.model_metadata_type import get_model_type
 
 from huggingface_hub import HfApi
 import huggingface_hub
@@ -38,17 +39,18 @@ size_pattern = re.compile(r"\d+(b|m)")
 def get_model_size(model_name, model_info):
     # In billions
     try:
-        return model_info.safetensors["total"] / 1e9
+        return round(model_info.safetensors["total"] / 1e9, 3) 
     except AttributeError:
         #print(f"Repository {model_id} does not have safetensors weights")
         pass
     try:
         size_match = re.search(size_pattern, model_name.lower())
         size = size_match.group(0)
-        return int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3
+        return round(int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3, 3)
     except AttributeError:
         return None
 
 
 def apply_metadata(leaderboard_data: List[dict]):
+    get_model_type(leaderboard_data)
     get_model_infos_from_hub(leaderboard_data)

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,163 @@
+from enum import Enum
+from typing import Dict, List
+
+class ModelType(Enum):
+    PT = "pretrained"
+    SFT = "finetuned"
+    RL = "with RL"
+
+
+TYPE_METADATA: Dict[str, ModelType] = {
+    "aisquared/dlite-v1-355m": ModelType.SFT,
+    "aisquared/dlite-v2-774m": ModelType.SFT,
+    "aisquared/dlite-v2-1_5b": ModelType.SFT,
+    "TheBloke/wizardLM-7B-HF": ModelType.SFT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.SFT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.SFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.SFT,
+    "wordcab/llama-natural-instructions-13b": ModelType.SFT,
+    "JosephusCheung/Guanaco": ModelType.SFT,
+    "AlekseyKorshuk/vicuna-7b": ModelType.SFT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.SFT,
+    "concedo/OPT-19M-ChatSalad": ModelType.SFT,
+    "digitous/Javalion-R": ModelType.SFT,
+    "digitous/Alpacino30b": ModelType.SFT,
+    "digitous/Javelin-GPTJ": ModelType.SFT,
+    "anton-l/gpt-j-tiny-random": ModelType.SFT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.SFT,
+    "gpt2-medium": ModelType.PT,
+    "PygmalionAI/pygmalion-6b": ModelType.SFT,
+    "medalpaca/medalpaca-7b": ModelType.SFT,
+    "medalpaca/medalpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-native": ModelType.SFT,
+    "chavinlo/gpt4-x-alpaca": ModelType.SFT,
+    "hakurei/lotus-12B": ModelType.SFT,
+    "amazon/LightGPT": ModelType.SFT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.SFT,
+    "mosaicml/mpt-7b": ModelType.PT,
+    "PSanni/Deer-3b": ModelType.SFT,
+    "bigscience/bloom-1b1": ModelType.PT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.SFT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.SFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.SFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.SFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.SFT,
+    "pillowtalks-ai/delta13b": ModelType.SFT,
+    "bigcode/starcoderplus": ModelType.SFT,
+    "microsoft/DialoGPT-large": ModelType.SFT,
+    "microsoft/CodeGPT-small-py": ModelType.SFT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.SFT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.SFT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.SFT,
+    "dvruette/llama-13b-pretrained-sft-epoch-2": ModelType.SFT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.SFT,
+    "openlm-research/open_llama_3b_350bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_7b_400bt_preview": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.SFT,
+    "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k": ModelType.SFT,
+    "databricks/dolly-v2-7b": ModelType.SFT,
+    "databricks/dolly-v2-3b": ModelType.SFT,
+    "databricks/dolly-v2-12b": ModelType.SFT,
+    "pinkmanlove/llama-65b-hf": ModelType.SFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.SFT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.SFT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.SFT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.SFT,
+    "eachadea/vicuna-7b-1.1": ModelType.SFT,
+    "eachadea/vicuna-13b": ModelType.SFT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.SFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.SFT,
+    "lamini/instruct-tuned-3b": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.SFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.SFT,
+    "CalderaAI/30B-Lazarus": ModelType.SFT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.SFT,
+    "distilgpt2": ModelType.PT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.SFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.SFT,
+    "junelee/wizard-vicuna-13b": ModelType.SFT,
+    "BreadAi/StoryPy": ModelType.SFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "Writer/camel-5b-hf": ModelType.SFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/lamini-neo-125m": ModelType.SFT,
+    "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4": ModelType.SFT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.SFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-1.3b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/galactica-120b": ModelType.PT,
+    "Abe13/jgpt2-v1": ModelType.SFT,
+    "gpt2-xl": ModelType.PT,
+    "HuggingFaceH4/stable-vicuna-13b-2904": ModelType.RL,
+    "HuggingFaceH4/llama-7b-ift-alpaca": ModelType.SFT,
+    "HuggingFaceH4/starchat-alpha": ModelType.SFT,
+    "HuggingFaceH4/starchat-beta": ModelType.SFT,
+    "ausboss/Llama30B-SuperHOT": ModelType.SFT,
+    "ausboss/llama-13b-supercot": ModelType.SFT,
+    "ausboss/llama-30b-supercot": ModelType.SFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.SFT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.SFT,
+    "victor123/WizardLM-13B-1.0": ModelType.SFT,
+    "AlpinDale/pygmalion-instruct": ModelType.SFT,
+    "tiiuae/falcon-7b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "cyl/awsome-llama": ModelType.SFT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.SFT,
+    "xzuyn/MedicWizard-7B": ModelType.SFT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.SFT,
+    "chainyo/alpaca-lora-7b": ModelType.SFT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-16B-multi": ModelType.SFT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.SFT,
+    "gpt2-large": ModelType.PT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "jondurbin/airoboros-7b": ModelType.SFT,
+    "jondurbin/airoboros-13b": ModelType.SFT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "NousResearch/Nous-Hermes-13b": ModelType.SFT,
+    "project-baize/baize-v2-7b": ModelType.SFT,
+    "project-baize/baize-v2-13b": ModelType.SFT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.SFT,
+    "LLMs/Vicuna-EvolInstruct-13B": ModelType.SFT,
+    "huggingtweets/jerma985": ModelType.SFT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.SFT,
+    "gpt2": ModelType.PT,
+    "alessandropalla/instruct_gpt2": ModelType.SFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.SFT,
+    "MayaPH/FinOPT-Franklin": ModelType.SFT,
+    "timdettmers/guanaco-33b-merged": ModelType.SFT,
+    "timdettmers/guanaco-65b-merged": ModelType.SFT,
+    "elinas/llama-30b-hf-transformers-4.29": ModelType.SFT,
+    "elinas/chronos-33b": ModelType.SFT,
+    "nmitchko/medguanaco-65b-GPTQ": ModelType.SFT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.SFT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.SFT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.SFT,
+    "nomic-ai/gpt4all-13b-snoozy": ModelType.SFT,
+    "nomic-ai/gpt4all-j": ModelType.SFT,
+}
+
+
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")

src/utils_display.py

@@ -7,6 +7,7 @@ class ColumnContent:
     name: str
     type: str 
     displayed_by_default: bool 
+    hidden: bool = False
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -14,8 +15,9 @@ def fields(raw_class):
 @dataclass(frozen=True)
 class AutoEvalColumn: # Auto evals column
     model = ColumnContent("Model", "markdown", True)
-    revision = ColumnContent("Revision", "str", True)
-    is_8bit = ColumnContent("8bit", "bool", False)
+    revision = ColumnContent("Revision", "str", True, True)
+    model_type = ColumnContent("Type", "bool", False)
+    is_8bit = ColumnContent("8bit", "bool", False, True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)
     likes = ColumnContent("Hub ❤️", "number", False)