Upload folder using huggingface_hub

app.py

@@ -30,7 +30,7 @@ use_real_api = False
 
 # Function to generate a paper_id using SHA-512 hash
 def generate_paper_id(paper_content):
-    return hashlib.sha512(paper_content).hexdigest()
+    return hashlib.sha512(paper_content.encode('utf-8')).hexdigest()
 
 # Function to get user IP address
 def get_user_ip():
@@ -165,6 +165,9 @@ def setup_interface():
     }
     """
     with gr.Blocks(css=css) as demo:
+        paper_content_state = gr.State()
+        model_a_state = gr.State()
+        model_b_state = gr.State()
         with gr.Tabs():
             with gr.TabItem("Reviewer Arena"):
                 gr.Markdown("## Reviewer Arena")
@@ -188,13 +191,13 @@ def setup_interface():
 
                 model_identity_message = gr.HTML("", visible=False)
 
-                def handle_vote_interface(vote, model_identity_message_a, model_identity_message_b, paper_content):
-                    return handle_vote(vote, model_identity_message_a, model_identity_message_b, paper_content)
+                def handle_vote_interface(vote, model_a, model_b, paper_content):
+                    return handle_vote(vote, model_a, model_b, paper_content)
 
                 submit_button.click(fn=review_papers, inputs=[file_input],
-                                    outputs=[review1, review2, vote, vote_button, model_identity_message, model_identity_message])
+                                    outputs=[review1, review2, vote, vote_button, model_a_state, model_b_state, paper_content_state])
 
-                vote_button.click(fn=handle_vote_interface, inputs=[vote, model_identity_message, model_identity_message],
+                vote_button.click(fn=handle_vote_interface, inputs=[vote, model_a_state, model_b_state, paper_content_state],
                                   outputs=[vote_message, vote, vote_button, another_paper_button])
 
                 another_paper_button.click(fn=lambda: None, inputs=None, outputs=None, js="() => { location.reload(); }")
@@ -204,7 +207,7 @@ def setup_interface():
 
                 # Fetch the leaderboard data from the database
                 leaderboard_data = get_leaderboard()
-                print(leaderboard_data)
+                # print(leaderboard_data)
                 
                 # Create the leaderboard HTML dynamically
                 leaderboard_html = """
@@ -224,19 +227,19 @@ def setup_interface():
                         <tbody>
                 """
                 
-                # for rank, model in enumerate(leaderboard_data, start=1):
-                #     leaderboard_html += f"""
-                #         <tr style="border: 1px solid #444; padding: 12px;">
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{rank}</td>
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['ModelID']}</td>
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['EloScore']}</td>
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">+3/-3</td> <!-- Adjust as needed -->
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['Votes']}</td>
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Organization</td> <!-- Add actual data if available -->
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">License</td> <!-- Add actual data if available -->
-                #             <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Knowledge Cutoff</td> <!-- Add actual data if available -->
-                #         </tr>
-                #     """
+                for rank, model in enumerate(leaderboard_data, start=1):
+                    leaderboard_html += f"""
+                        <tr style="border: 1px solid #444; padding: 12px;">
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{rank}</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['ModelID']}</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['EloScore']}</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['CI_Lower']} - {model['CI_Upper']}</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['Votes']}</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Organization</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">License</td>
+                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Knowledge Cutoff</td>
+                        </tr>
+                    """
 
                 leaderboard_html += """
                         </tbody>
@@ -253,4 +256,4 @@ def setup_interface():
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     demo = setup_interface()
-    demo.launch()
+    demo.launch()

aws_utils.py

@@ -2,6 +2,7 @@ import boto3
 import uuid
 import datetime
 import os
+from decimal import Decimal, getcontext
 from dotenv import load_dotenv
 
 try:
@@ -27,7 +28,7 @@ leaderboards_table = dynamodb.Table('reviewer_arena_leaderboard')
 # Function to write a request to the Requests table
 def write_request(user_id, paper_id, model_a, model_b, vote):
     request_id = str(uuid.uuid4())
-    timestamp = datetime.datetime.now().isoformat()
+    timestamp = str(Decimal(datetime.datetime.now().timestamp()))
     
     response = requests_table.put_item(
         Item={
@@ -44,55 +45,115 @@ def write_request(user_id, paper_id, model_a, model_b, vote):
 
 # Function to update leaderboard after a vote
 def update_leaderboard(model_a, model_b, vote):
+    # Map vote options to simpler keys
+    vote_mapping = {
+        "👍 A is better": "A is better",
+        "👍 B is better": "B is better",
+        "👔 Tie": "Tie",
+        "👎 Both are bad": "Tie"  # Assuming "Both are bad" is treated as a tie
+    }
+    vote = vote_mapping.get(vote, "Tie")  # Default to "Tie" if vote is not found
+
     # Retrieve current stats for ModelA and ModelB
     model_a_stats = leaderboards_table.get_item(Key={'ModelID': model_a}).get('Item', {})
     model_b_stats = leaderboards_table.get_item(Key={'ModelID': model_b}).get('Item', {})
     
     # Initialize stats if they don't exist
     if not model_a_stats:
-        model_a_stats = {'ModelID': model_a, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
+        model_a_stats = {'ModelID': model_a, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': Decimal(1200), 'Votes': 0}
+        leaderboards_table.put_item(Item=model_a_stats)
     if not model_b_stats:
-        model_b_stats = {'ModelID': model_b, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
+        model_b_stats = {'ModelID': model_b, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': Decimal(1200), 'Votes': 0}
+        leaderboards_table.put_item(Item=model_b_stats)
     
     # Update stats based on the vote
-    if vote == "A is better":
-        model_a_stats['Wins'] += 1
-        model_b_stats['Losses'] += 1
-    elif vote == "B is better":
-        model_a_stats['Losses'] += 1
-        model_b_stats['Wins'] += 1
-    elif vote == "Tie":
-        model_a_stats['Ties'] += 1
-        model_b_stats['Ties'] += 1
-    model_a_stats['Votes'] += 1
-    model_b_stats['Votes'] += 1
+    update_expressions = {
+        "A is better": {
+            "model_a": "SET Wins = Wins + :inc, Votes = Votes + :inc",
+            "model_b": "SET Losses = Losses + :inc, Votes = Votes + :inc"
+        },
+        "B is better": {
+            "model_a": "SET Losses = Losses + :inc, Votes = Votes + :inc",
+            "model_b": "SET Wins = Wins + :inc, Votes = Votes + :inc"
+        },
+        "Tie": {
+            "model_a": "SET Ties = Ties + :inc, Votes = Votes + :inc",
+            "model_b": "SET Ties = Ties + :inc, Votes = Votes + :inc"
+        }
+    }
     
-    # Calculate new Elo scores (simple Elo calculation for illustration)
-    model_a_stats['EloScore'], model_b_stats['EloScore'] = calculate_elo(model_a_stats['EloScore'], model_b_stats['EloScore'], vote)
+    expression_a = update_expressions[vote]["model_a"]
+    expression_b = update_expressions[vote]["model_b"]
+    
+    # Update ModelA stats
+    leaderboards_table.update_item(
+        Key={'ModelID': model_a},
+        UpdateExpression=expression_a,
+        ExpressionAttributeValues={':inc': 1}
+    )
+    
+    # Update ModelB stats
+    leaderboards_table.update_item(
+        Key={'ModelID': model_b},
+        UpdateExpression=expression_b,
+        ExpressionAttributeValues={':inc': 1}
+    )
     
-    # Write updated stats back to the Leaderboards table
-    leaderboards_table.put_item(Item=model_a_stats)
-    leaderboards_table.put_item(Item=model_b_stats)
+    # Calculate new Elo scores (simple Elo calculation for illustration)
+    new_elo_a, new_elo_b = calculate_elo(model_a_stats['EloScore'], model_b_stats['EloScore'], vote)
+
+    # Calculate 95% CI for new Elo scores
+    ci_a_lower, ci_a_upper = calculate_95_ci(new_elo_a, model_a_stats['Votes'] + 1)
+    ci_b_lower, ci_b_upper = calculate_95_ci(new_elo_b, model_b_stats['Votes'] + 1)
+
+    # Update Elo scores and 95% CI
+    leaderboards_table.update_item(
+        Key={'ModelID': model_a},
+        UpdateExpression="SET EloScore = :new_elo, CI_Lower = :ci_lower, CI_Upper = :ci_upper",
+        ExpressionAttributeValues={':new_elo': Decimal(new_elo_a), ':ci_lower': Decimal(ci_a_lower), ':ci_upper': Decimal(ci_a_upper)}
+    )
+
+    leaderboards_table.update_item(
+        Key={'ModelID': model_b},
+        UpdateExpression="SET EloScore = :new_elo, CI_Lower = :ci_lower, CI_Upper = :ci_upper",
+        ExpressionAttributeValues={':new_elo': Decimal(new_elo_b), ':ci_lower': Decimal(ci_b_lower), ':ci_upper': Decimal(ci_b_upper)}
+    )
+
+# Set the precision for Decimal
+getcontext().prec = 28
 
 # Function to calculate new Elo scores
 def calculate_elo(elo_a, elo_b, vote, k=32):
-    expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
-    expected_b = 1 / (1 + 10 ** ((elo_a - elo_b) / 400))
+    # Ensure elo_a and elo_b are Decimals
+    elo_a = Decimal(elo_a)
+    elo_b = Decimal(elo_b)
+    
+    expected_a = 1 / (1 + Decimal(10) ** ((elo_b - elo_a) / Decimal(400)))
+    expected_b = 1 / (1 + Decimal(10) ** ((elo_a - elo_b) / Decimal(400)))
     
     if vote == "A is better":
-        actual_a = 1
-        actual_b = 0
+        actual_a = Decimal(1)
+        actual_b = Decimal(0)
     elif vote == "B is better":
-        actual_a = 0
-        actual_b = 1
+        actual_a = Decimal(0)
+        actual_b = Decimal(1)
     else:  # Tie
-        actual_a = 0.5
-        actual_b = 0.5
+        actual_a = Decimal(0.5)
+        actual_b = Decimal(0.5)
     
-    new_elo_a = elo_a + k * (actual_a - expected_a)
-    new_elo_b = elo_b + k * (actual_b - expected_b)
+    new_elo_a = elo_a + Decimal(k) * (actual_a - expected_a)
+    new_elo_b = elo_b + Decimal(k) * (actual_b - expected_b)
     
-    return round(new_elo_a), round(new_elo_b)
+    return round(new_elo_a, 2), round(new_elo_b, 2)
+
+# Function to calculate 95% CI for Elo scores
+def calculate_95_ci(elo, votes, z=1.96):
+    if votes == 0:
+        return Decimal(0), Decimal(0)
+    elo = Decimal(elo)  # Ensure elo is a Decimal
+    std_error = Decimal(400) / (Decimal(votes).sqrt())
+    margin = Decimal(z) * std_error
+    return round(elo - margin, 2), round(elo + margin, 2)
 
 # Function to query leaderboard
 def get_leaderboard():