Clean up master branch

.gitignore

@@ -1,4 +1,12 @@
+# OS-related
 **/.DS_Store
+
+# Environment
 .envrc
+
+# Editor
 pyrightconfig.json
 .idea
+
+# Data
+data/

app.py

@@ -1,99 +0,0 @@
-import gradio as gr
-import requests
-import pandas as pd
-from huggingface_hub.hf_api import SpaceInfo
-import matplotlib.pyplot as plt
-import plotly.express as px
-
-model_perf_table = "data/test.csv"
-logo_path = "img/image.png"
-
-
-def get_blocks_party_spaces():
-    df = pd.read_csv(model_perf_table)
-    df = df.sort_values(by=['score'],ascending=False)
-    return df
-
-
-def get_blocks_party_spaces_with_formula(formula=None):
-    # get the dataframe
-    df = get_blocks_party_spaces()
-    if formula:
-        try:
-            df[str(formula)] = df.eval(formula)
-        except:
-            pass # Handle this error properly in your code
-    return df
-
-def create_scatter(x, y, z):
-    df = get_blocks_party_spaces()
-    if z is None or z == 'None' or z == '':
-        fig, ax = plt.subplots()
-
-        ax.scatter(list(df[x]),list(df[y]), marker='o', s=50, c='blue')
-        for i, label in enumerate(list(df['model'])):
-            ax.text(list(df[x])[i],list(df[y])[i],str(label))
-        ax.set_xlabel(x)
-        ax.set_ylabel(y)
-
-        ax.spines['top'].set_visible(False)
-        ax.spines['right'].set_visible(False)
-        plt.grid(axis='y', linestyle='--', alpha=0.7)
-
-    else:
-        fig = px.scatter_3d(df, x=x, y=y, z=z, text=df['model'])
-
-        # Set axis labels and title
-        fig.update_layout(scene=dict(
-            xaxis_title=x,
-            yaxis_title=y,
-            zaxis_title=z,
-        ),
-            title='3D Scatter Plot'
-        )
-
-    return fig
-
-block = gr.Blocks()
-with block:
-    # gr.outputs.HTML(f'<img src="{logo_path}" alt="logo" height="1000px">')
-    # img = gr.Image(logo_path,shape=[1,2]).style( rounded=False)
-
-    gr.Markdown(f"""
-                # 🦙💦SpitFight - Leaderboard for LLM 
-                """)
-    with gr.Tabs():
-        with gr.TabItem("Leaderboard"):
-            with gr.Row():
-                data = gr.outputs.Dataframe(type="pandas")
-            with gr.Row():
-                formula_input = gr.inputs.Textbox(lines=1, label="User Designed Column", placeholder = 'e.g. verbosity/latency')
-                data_run = gr.Button("Add To Table")
-                data_run.click(get_blocks_party_spaces_with_formula, inputs=formula_input, outputs=data)
-            # running the function on page load in addition to when the button is clicked
-            with gr.Row():
-                with gr.Column():
-                    scatter_input = [gr.inputs.Dropdown(choices=get_blocks_party_spaces().columns.tolist()[1:], label="X-axis"),
-                         gr.inputs.Dropdown(choices=get_blocks_party_spaces().columns.tolist()[1:], label="Y-axis"),
-                         gr.inputs.Dropdown(choices=[None]+get_blocks_party_spaces().columns.tolist()[1:], label="Z-axis (Optional)")]
-                    fig_run = gr.Button("Generate Figure")
-
-                with gr.Column():
-                    gen_figure = gr.Plot()# gr.outputs.Image(type="pil")
-                    fig_run.click(create_scatter, inputs=scatter_input, outputs=gen_figure)
-
-
-        with gr.TabItem("About"):
-            gr.Markdown(f"""
-                        ## Metrics: 
-                        - **Human Score**: The average score given by human evaluators.
-                        - **Throughput**: The number of tokens generated per second.
-                        - **Verbosity**: The average number of generated tokens in the model's response.
-                        - **Latency**: The average time it takes for the model to generate a response.
-                        - **Memory**: The base memory usage of the model.
-                        """)
-
-    block.load(get_blocks_party_spaces_with_formula, inputs=None, outputs=data)
-
-block.launch(share=True)
-# block.launch( )

benchmark.py

@@ -41,6 +41,7 @@ SYSTEM_PROMPTS = {
 def main(
     model_path: str,
     input_file: str,
+    output_dir: str = "data",
     device_index: int = 0,
     task: Literal[tuple(SYSTEM_PROMPTS)] = "chat",  # type: ignore
     load_8bit: bool = False,
@@ -48,20 +49,18 @@ def main(
     repitition_penalty: float = 1.0,
     max_new_tokens: int = 512,
 ) -> None:
-    """Run the main routine.
-
-    Code structure is based on
-    https://github.com/lm-sys/FastChat/blob/57dea54055/fastchat/serve/inference.py#L249
+    """Run benchmarking for one model on the entire input file.
 
     Args:
         model_path: Path to or Huggingface Hub Id of the model.
         input_file: Path to the input JSON file. Assumed to be our cleaned ShareGPT data.
-        device_index: Index of the GPU to use for inference.
-        task: Type of task to perform inference on.
-        load_8bit: Whether to load the model in 8-bit mode.
-        temperature: Temperature to use for sampling.
-        repitition_penalty: Repitition penalty to use for the model.
-        max_new_tokens: Maximum numbers of tokens to generate, ignoring the prompt.
+        output_dir: Path to the output directory. (Default: "data")
+        device_index: Index of the GPU to use for inference. (Default: 0)
+        task: Type of task to perform inference on. (Default: "chat")
+        load_8bit: Whether to load the model in 8-bit mode. (Default: False)
+        temperature: Temperature to use for sampling. (Default: 0.7)
+        repitition_penalty: Repitition penalty to use for the model. (Default: 1.0)
+        max_new_tokens: Maximum numbers of tokens to generate, ignoring the prompt. (Default: 512)
     """
     # NOTE(JW): ChatGLM is implemented as a special case in FastChat inference.
     # Also, it's primarily a model that's fine-tuned for Chinese, so it doesn't
@@ -76,7 +75,7 @@ def main(
     if model_path.endswith("/"):
         model_path = model_path[:-1]
     model_name_cleaned = "--".join(model_path.split("/")[-2:])
-    output_dir = f"data/{task}/{model_name_cleaned}"
+    output_dir = f"{output_dir}/{task}/{model_name_cleaned}"
     output_csv_path = f"{output_dir}/benchmark.json"
     config_json_path = f"{output_dir}/config.json"
     table = Table(title="Benchmark")

data/chat/lmsys--vicuna-7B/benchmark.json

@@ -1,38 +0,0 @@
-[
-{
-    "model": "lmsys--vicuna-7B",
-    "throughput": 30.558443157709224,
-    "response_length": 331,
-    "latency": 10.831703640520573,
-    "energy": 2215.2789999842644,
-    "input": "A chat between a human user (prompter) and an artificial intelligence (AI) assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: Summarize the main ideas of Jeff Walker's Product Launch Formula into bullet points as it pertains to a growth marketing agency implementing these strategies and tactics for their clients... ASSISTANT:",
-    "output": "Sure, here are the main ideas of Jeff Walker's Product Launch Formula and how they can be applied to a growth marketing agency implementing these strategies and tactics for their clients:\n\n1. Pre-launch phase:\n* Build an audience through content marketing, social media, and other channels.\n* Create a sense of urgency and exclusivity by offering early access or special bonuses to those on a mailing list.\n* Use teasers, sneak peeks, and other forms of anticipation to build excitement.\n2. Launch phase:\n* Use a webinar or live event to launch the product, using stories, testimonials, and social proof to build trust and credibility.\n* Offer a limited-time special offer or bonus to those who purchase during the launch.\n* Continually communicate with the audience during the launch phase, answering questions, providing updates, and building relationships.\n3. Post-launch phase:\n* Follow up with customers to make sure they are satisfied and provide additional support and resources.\n* Use customer testimonials, success stories, and case studies to continue building social proof and credibility.\n* Continually update and improve the product based on customer feedback.\n\nOverall, the Product Launch Formula emphasizes the importance of building an engaged audience, creating urgency and exclusivity, and providing ongoing support and value to customers. By implementing these strategies and tactics, a growth marketing agency can help their clients launch successful products and grow their businesses."
-},
-{
-    "model": "lmsys--vicuna-7B",
-    "throughput": 31.514563598172494,
-    "response_length": 98,
-    "latency": 3.109673395752907,
-    "energy": 640.4210000038147,
-    "input": "A chat between a human user (prompter) and an artificial intelligence (AI) assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: How to tell if a customer segment is well segmented? In 3 bullet points. ASSISTANT:",
-    "output": "1. Homogeneity: The segment should consist of customers who share similar characteristics, behaviors, or needs.\n2. Distinctiveness: The segment should be different from other segments in terms of their characteristics, behaviors, or needs.\n3. Stability: The segment should remain relatively consistent over time and not change drastically. If the segment is constantly changing or becoming more similar to other segments, it may indicate that the segmentation is not effective."
-},
-{
-    "model": "lmsys--vicuna-7B",
-    "throughput": 30.362787676034074,
-    "response_length": 346,
-    "latency": 11.39552809484303,
-    "energy": 2334.6629999876022,
-    "input": "A chat between a human user (prompter) and an artificial intelligence (AI) assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: In Java, I want to replace string like \"This is a new {object} at {place}\" with a Map, {object: \"student\", \"point 3, 4\"}, and get a result \"This is a new student at point 3, 4\". How can I do? ASSISTANT:",
-    "output": "To replace the string with a map, you can use the `String.format()` method and pass in an object that contains the key-value pairs of the map. Here's an example of how you can do this in Java:\n```\nString originalString = \"This is a new {object} at {place}\";\nString formattedString = String.format(\"This is a new %s at %s\",\n                                         new ObjectMapper().writeValueAsString(map),\n                                         new ObjectMapper().writeValueAsString(place));\n```\nIn this example, `originalString` is the original string that you want to replace, `map` is the map that you want to use to replace the {object} and {place} placeholders, and `ObjectMapper` is a class that is used to convert the map to a string. The `writeValueAsString()` method is used to convert the map to a JSON string, which can then be passed to the `String.format()` method to replace the placeholders in the string.\n\nTo get the result \"This is a new student at point 3, 4\", you can use the following code:\n```\nString result = String.format(\"This is a new %s at %s\",\n                               new ObjectMapper().writeValueAsString(map),\n                               new ObjectMapper().writeValueAsString(place));\n```\nYou can then pass the `result` string to a `println` statement or use it in any other way that you like.\n\nI hope this helps! Let me know if you have any other questions."
-},
-{
-    "model": "lmsys--vicuna-7B",
-    "throughput": 31.208981333084907,
-    "response_length": 96,
-    "latency": 3.076037598773837,
-    "energy": 649.0310000181198,
-    "input": "A chat between a human user (prompter) and an artificial intelligence (AI) assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: Metaphorical language is also used to describe the various addressing modes of the instructions. Grandiose language to express their excitement and admiration for the functionality of the instructions being described. Now, rewrite this with more perplexity:\n\nJMP ABCD\nMOV AX, [BX+SI]\nMOV AX, [100]\nMOV AX, [BX]\nMOV AX, [BX\\*2+SI]\nMOV AX, BX\nMOV AX, 7 ASSISTANT:",
-    "output": "The instructions described seem to possess a remarkable array of addressing modes, each with its own unique set of characteristics. The use of grandiose language to express excitement and admiration for these instructions is certainly understandable, as their functionality seems to transcend the mere execution of basic computations. With so many different addressing modes to choose from, one might be forgiven for feeling a sense of perplexity and wonder at the sheer complexity of these instructions."
-}
-]

data/chat/lmsys--vicuna-7B/config.json

@@ -1,10 +0,0 @@
-{
-    "model_path": "/data/leaderboard/weights/lmsys/vicuna-7B",
-    "input_file": "/data/leaderboard/sharegpt/sg_90k_part1_html_cleaned_lang_first_sampled.json",
-    "device_index": 0,
-    "task": "chat",
-    "load_8bit": false,
-    "temperature": 0.7,
-    "repitition_penalty": 1.0,
-    "max_new_tokens": 512
-}

data/test.csv

@@ -1,6 +0,0 @@
-model,score,throughput,response_length,latency,energy
-lmsys/vicuna-13B,1054,19.47,178.71,10.37,12
-lmsys/vicuna-7B,1007,31.26,217.33,7.71,32
-metaai/llama-13B,854,18.49,390.9,22.74,24
-lmsys/fastchat-t5-3B,941,31.98,131.12,6.09,4
-BlinkDL/rwkv-7B,928,23.09,239.98,9.62,12