import gradio as gr
from utils import process_paper
import os
import logging
import html
from logging_config import setup_logging
from aws_utils import update_leaderboard, get_leaderboard, write_request
from flask import request
import hashlib
import uuid
import json
from dotenv import load_dotenv
try:
load_dotenv()
except:
pass
setup_logging()
paper_dir = 'path_to_temp_storage'
prompt_dir = 'iclr2024'
api_keys = {
'openai_api_key': os.environ.get('OPENAI_API_KEY'),
'claude_api_key': os.environ.get('ANTHROPIC_API_KEY'),
'gemini_api_key': os.environ.get('GEMINI_API_KEY'),
'commandr_api_key': os.environ.get('COMMANDR_API_KEY')
}
##
use_real_api = True
# Function to generate a paper_id using SHA-512 hash
def generate_paper_id(paper_content):
return hashlib.sha512(paper_content.encode('utf-8')).hexdigest()
# Function to get user IP address
def get_user_ip():
try:
return request.remote_addr
except:
return "Unknown"
def review_papers(pdf_file):
logging.info(f"Received file type: {type(pdf_file)}")
paper_content = pdf_file.read() if hasattr(pdf_file, 'read') else pdf_file # Read the content of the uploaded PDF file
if use_real_api:
reviews, selected_models = process_paper(
pdf_file, paper_dir, prompt_dir, api_keys)
processed_reviews = []
for review in reviews:
processed_review = {}
for section in review:
if ':' in section:
key, value = section.split(':', 1)
processed_value = value.strip().replace('\n', '
')
processed_review[key.strip()] = html.escape(
processed_value)
processed_reviews.append(processed_review)
reviews = processed_reviews
else:
reviews = [
{
"Summary": "This is a placeholder review for Model 2. The paper explores advanced methodologies in reinforcement learning applied to autonomous driving systems, proposing significant enhancements to decision-making algorithms that could improve safety and operational efficiency. The authors provide a detailed analysis of the current limitations of existing systems and suggest innovative solutions that could transform the field.",
"Soundness": "The assumptions underlying the proposed enhancements are occasionally not fully justified, particularly concerning the scalability of the algorithms under varied and unpredictable traffic conditions. A more rigorous examination of these assumptions is necessary to solidify the paper's foundation.",
"Presentation": "While the paper is structured adequately, some sections delve into technical details that are not sufficiently elucidated for a broader audience. This could potentially limit the paper's impact and accessibility, making it challenging for non-specialists to fully grasp the implications of the research.",
"Contribution": "The paper makes a moderate contribution to the existing body of knowledge, offering incremental improvements over current methodologies rather than a completely novel approach. However, these improvements are significant and could lead to better practical implementations in the field of autonomous driving.",
"Strengths": "The initial results presented in the paper are promising, showing potential for the proposed methods. The inclusion of real-world data in the preliminary experiments adds a layer of credibility and relevance to the results, showcasing the practical applicability of the research.",
"Weaknesses": "The paper lacks detailed exposition on the methodology, particularly in how the algorithms adapt to unexpected or novel scenarios. This is a critical area that requires further development and testing to ensure the robustness and reliability of the proposed solutions.",
"Questions/Suggestions": "The statistical analysis section could be enhanced by incorporating more robust statistical techniques and a wider array of metrics. Additionally, conducting tests in a variety of driving environments could help in substantiating the claims made and strengthen the overall findings of the research.",
"Ethics Review": "The research complies with all ethical standards, addressing potential ethical issues related to autonomous driving comprehensively. Issues such as privacy concerns, decision-making in critical situations, and the overall impact on societal norms are discussed and handled with the utmost care.",
"Overall Score": "3/5",
"Confidence": "Confidence in the findings is moderate. While the initial results are encouraging, the limited scope of testing and some unresolved questions regarding scalability and robustness temper the confidence in these results.",
"Code of Conduct": "There are no violations of the code of conduct noted. The research upholds ethical standards and maintains transparency in methodologies and data usage, contributing to its integrity and the trustworthiness of the findings."
},
{
"Summary": "This is a placeholder review for Model 2. The paper explores advanced methodologies in reinforcement learning applied to autonomous driving systems, proposing significant enhancements to decision-making algorithms that could improve safety and operational efficiency. The authors provide a detailed analysis of the current limitations of existing systems and suggest innovative solutions that could transform the field.",
"Soundness": "The assumptions underlying the proposed enhancements are occasionally not fully justified, particularly concerning the scalability of the algorithms under varied and unpredictable traffic conditions. A more rigorous examination of these assumptions is necessary to solidify the paper's foundation.",
"Presentation": "While the paper is structured adequately, some sections delve into technical details that are not sufficiently elucidated for a broader audience. This could potentially limit the paper's impact and accessibility, making it challenging for non-specialists to fully grasp the implications of the research.",
"Contribution": "The paper makes a moderate contribution to the existing body of knowledge, offering incremental improvements over current methodologies rather than a completely novel approach. However, these improvements are significant and could lead to better practical implementations in the field of autonomous driving.",
"Strengths": "The initial results presented in the paper are promising, showing potential for the proposed methods. The inclusion of real-world data in the preliminary experiments adds a layer of credibility and relevance to the results, showcasing the practical applicability of the research.",
"Weaknesses": "The paper lacks detailed exposition on the methodology, particularly in how the algorithms adapt to unexpected or novel scenarios. This is a critical area that requires further development and testing to ensure the robustness and reliability of the proposed solutions.",
"Questions/Suggestions": "The statistical analysis section could be enhanced by incorporating more robust statistical techniques and a wider array of metrics. Additionally, conducting tests in a variety of driving environments could help in substantiating the claims made and strengthen the overall findings of the research.",
"Ethics Review": "The research complies with all ethical standards, addressing potential ethical issues related to autonomous driving comprehensively. Issues such as privacy concerns, decision-making in critical situations, and the overall impact on societal norms are discussed and handled with the utmost care.",
"Overall Score": "3/5",
"Confidence": "Confidence in the findings is moderate. While the initial results are encouraging, the limited scope of testing and some unresolved questions regarding scalability and robustness temper the confidence in these results.",
"Code of Conduct": "There are no violations of the code of conduct noted. The research upholds ethical standards and maintains transparency in methodologies and data usage, contributing to its integrity and the trustworthiness of the findings."
}
]
selected_models = ['model1-placeholder', 'model2-placeholder']
review_texts = []
for review in reviews:
formatted_review = "
Thank you for your vote!
Model A: {model_a}
Model B: {model_b}
" return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True) def setup_interface(): logging.debug("Setting up Gradio interface.") css = """ .review-container { padding: 10px; margin-bottom: 20px; border: 1px solid #ccc; background-color: #f9f9f9; } .review-section { margin-bottom: 12px; padding: 8px; background-color: #ffffff; border-left: 4px solid #007BFF; padding-left: 10px; } .review-section strong { color: #333; font-weight: bold; display: block; margin-bottom: 5px; } .review-section span, .gr-markdown { color: #000; font-size: 14px; line-height: 1.5; display: block; white-space: normal; opacity: 1; } .model-label { font-size: 18px; font-weight: bold; color: #007BFF; margin-bottom: 10px; } .gr-file, .gr-button, .gr-radio { width: 300px; margin: auto; } .gr-button-small { width: 150px; height: 40px; font-size: 16px; } """ with gr.Blocks(css=css) as demo: paper_content_state = gr.State() model_a_state = gr.State() model_b_state = gr.State() with gr.Tabs(): with gr.TabItem("Reviewer Arena"): gr.Markdown("## Reviewer Arena") gr.Markdown("We serve the academic community by pre-reviewing academic papers in real-time and helping improve manuscripts before submission to conferences or journals. Reviewer Arena evaluates LLM reviewer quality based on preferences by direct and anonymous comparison of reviews.") with gr.Row(): file_input = gr.File(label="Upload Academic Paper") submit_button = gr.Button("Submit!", elem_id="submit-button") with gr.Row(): with gr.Column(): gr.HTML("Rank | Model | Arena Elo | 95% CI | Votes | Organization | License | Knowledge Cutoff |
---|---|---|---|---|---|---|---|
{rank} | {model['ModelID']} | {model['EloScore']} | {model['CI_Lower']} - {model['CI_Upper']} | {model['Votes']} | {model['Organization']} | {model['License']} | {model['KnowledgeCutoff']} |
Rank | Model | Arena Elo | 95% CI | Votes | Organization | License | Knowledge Cutoff |
---|---|---|---|---|---|---|---|
{rank} | {model['ModelID']} | {model['EloScore']} | {model['CI_Lower']} - {model['CI_Upper']} | {model['Votes']} | {model['Organization']} | {model['License']} | {model['KnowledgeCutoff']} |