src/about.py

from dataclasses import dataclass
from enum import Enum


@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str
    category: str


# Select your tasks here
# ---------------------------------------------------
class Tasks(Enum):
    task0 = Task("FPB", "F1", "FPB", category="Textual Analysis (TA)")
    task2 = Task("FiQA-SA", "F1", "FiQA-SA", category="Textual Analysis (TA)")
    task3 = Task("TSA", "RMSE", "TSA", category="Textual Analysis (TA)")
    task4 = Task("Headlines", "AvgF1", "Headlines", category="Textual Analysis (TA)")
    task5 = Task("FOMC", "F1", "FOMC", category="Textual Analysis (TA)")
    task7 = Task("FinArg-ACC", "MicroF1", "FinArg-ACC", category="Textual Analysis (TA)")
    task8 = Task("FinArg-ARC", "MicroF1", "FinArg-ARC", category="Textual Analysis (TA)")
    task9 = Task("MultiFin", "MicroF1", "MultiFin", category="Textual Analysis (TA)")
    task10 = Task("MA", "MicroF1", "MA", category="Textual Analysis (TA)")
    task11 = Task("MLESG", "MicroF1", "MLESG", category="Textual Analysis (TA)")
    task12 = Task("NER", "EntityF1", "NER", category="Information Extraction (IE)")
    task13 = Task("FINER-ORD", "EntityF1", "FINER-ORD", category="Information Extraction (IE)")
    task14 = Task("FinRED", "F1", "FinRED", category="Information Extraction (IE)")
    task15 = Task("SC", "F1", "SC", category="Information Extraction (IE)")
    task16 = Task("CD", "F1", "CD", category="Information Extraction (IE)")
    task17 = Task("FinQA", "EmAcc", "FinQA", category="Question Answering (QA)")
    task18 = Task("TATQA", "EmAcc", "TATQA", category="Question Answering (QA)")
    task19 = Task("ConvFinQA", "EmAcc", "ConvFinQA", category="Question Answering (QA)")
    task20 = Task("FNXL", "EntityF1", "FNXL", category="Information Extraction (IE)")
    task21 = Task("FSRL", "EntityF1", "FSRL", category="Information Extraction (IE)")
    task22 = Task("EDTSUM", "Rouge-1", "EDTSUM", category="Text Generation (TG)")
    task25 = Task("ECTSUM", "Rouge-1", "ECTSUM", category="Text Generation (TG)")
    task28 = Task("BigData22", "Acc", "BigData22", category="Forecasting (FO)")
    task30 = Task("ACL18", "Acc", "ACL18", category="Forecasting (FO)")
    task32 = Task("CIKM18", "Acc", "CIKM18", category="Forecasting (FO)")
    task34 = Task("German", "MCC", "German", category="Risk Management (RM)")
    task36 = Task("Australian", "MCC", "Australian", category="Risk Management (RM)")
    task38 = Task("LendingClub", "MCC", "LendingClub", category="Risk Management (RM)")
    task40 = Task("ccf", "MCC", "ccf", category="Risk Management (RM)")
    task42 = Task("ccfraud", "MCC", "ccfraud", category="Risk Management (RM)")
    task44 = Task("polish", "MCC", "polish", category="Risk Management (RM)")
    task46 = Task("taiwan", "MCC", "taiwan", category="Risk Management (RM)")
    task48 = Task("portoseguro", "MCC", "portoseguro", category="Risk Management (RM)")
    task50 = Task("travelinsurance", "MCC", "travelinsurance", category="Risk Management (RM)")
    task51 = Task("MultiFin-ES", "F1", "MultiFin-ES", category="Spanish")
    task52 = Task("EFP", "F1", "EFP", category="Spanish")
    task53 = Task("EFPA", "F1", "EFPA", category="Spanish")
    task54 = Task("FinanceES", "F1", "FinanceES", category="Spanish")
    task55 = Task("TSA-Spanish", "F1", "TSA-Spanish", category="Spanish")

NUM_FEWSHOT = 0  # Change with your few shot
# ---------------------------------------------------


# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">🐲 Open Financial LLM Leaderboard</h1>"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
🌟 The Open Financial LLM Leaderboard: Evaluate and compare the performance of financial Large Language Models (LLMs).

When you submit a model on the "Submit here!" page, it is automatically evaluated on a set of financial benchmarks.

The GPU used for evaluation is operated with the support of __[Wuhan University](http://en.whu.edu.cn/)__ and __[University of Florida](https://www.ufl.edu/)__.

The datasets used for evaluation consist of diverse financial datasets from `FinBen` benchmark to assess tasks such as sentiment analysis, named entity recognition, question answering, and more.

More details about the benchmarks and the evaluation process are provided on the “About” page.
"""

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = """
## Introduction

The **Open Financial LLMs Leaderboard (OFLL)** is meticulously designed to rigorously track, rank, and evaluate state-of-the-art models in financial Natural Language Understanding and Prediction. Our leaderboard not only covers standard NLP tasks but also incorporates financial prediction tasks such as stock movement and credit scoring, offering a comprehensive evaluation for real-world financial applications.

## Icons & Model Types

- 🟢 : pretrained or continuously pretrained
- 🔶 : fine-tuned on domain-specific datasets
- 💬 : chat models (RLHF, DPO, ORPO, ...)
- 🤝 : base merges and moerges

If the icon is "?", it indicates that there is insufficient information about the model. Please provide information about the model through an issue! 🤩

**Note 1**: We reserve the right to correct any incorrect tags/icons after manual verification to ensure the accuracy and reliability of the leaderboard.

**Note 2** ⚠️: Some models might be widely discussed as subjects of caution by the community, implying that users should exercise restraint when using them. Models that have used the evaluation set for training to achieve a high leaderboard ranking, among others, may be selected as subjects of caution and might result in their deletion from the leaderboard.

## How It Works

📈 We evaluate models using Pixiu, a powerful and straightforward framework to test and assess language models on a large number of different evaluation tasks from FinBen, using datasets validated by financial experts.

### Evaluation Metrics

Our evaluation metrics include, but are not limited to, Accuracy, F1 Score, ROUGE score, BERTScore, and Matthews correlation coefficient (MCC), providing a multidimensional assessment of model performance. Metrics for specific tasks are as follows:

- **FPB**: F1, Accuracy. Financial PhraseBank classification task.
- **FiQA-SA**: F1. Sentiment analysis on FiQA financial domain.
- **TSA**: F1, Accuracy. Sentiment analysis.
- **Headlines**: AvgF1. News headline classification.
- **FOMC**: F1, Accuracy. Hawkish-dovish classification.
- **FinArg-ACC**: F1, Accuracy. Financial argument unit classification.
- **FinArg-ARC**: F1, Accuracy. Financial argument relation classification.
- **MultiFin**: F1, Accuracy. Multi-class financial sentiment analysis.
- **MA**: F1, Accuracy. Deal completeness classification.
- **MLESG**: F1, Accuracy. ESG issue identification.
- **NER**: EntityF1. Named entity recognition in financial texts.
- **FINER-ORD**: EntityF1. Ordinal classification in financial NER.
- **FinRED**: F1, EntityF1. Financial relation extraction from text.
- **SC**: F1, EntityF1. Causal classification task in the financial domain.
- **CD**: F1, EntityF1. Causal detection.
- **FinQA**: EmAcc. Numerical question answering in finance.
- **TATQA**: F1, EmAcc. Table-based question answering in financial documents.
- **ConvFinQA**: EmAcc. Multi-turn question answering in finance.
- **FNXL**: F1, EmAcc. Numeric labeling in financial texts.
- **FSRL**: F1, EmAcc. Financial statement relation linking.
- **EDTSUM**: ROUGE, BERTScore, BARTScore. Extractive document summarization in finance.
- **ECTSUM**: ROUGE, BERTScore, BARTScore. Extractive content summarization.
- **BigData22**: Accuracy, MCC. Stock movement prediction.
- **ACL18**: Accuracy, MCC. Financial news-based stock prediction.
- **CIKM18**: Accuracy, MCC. Financial market prediction using news.
- **German**: F1, MCC. Credit scoring in the German market.
- **Australian**: F1, MCC. Credit scoring in the Australian market.
- **LendingClub**: F1, MCC. Peer-to-peer lending risk prediction.
- **ccf**: F1, MCC. Credit card fraud detection.
- **ccfraud**: F1, MCC. Credit card transaction fraud detection.
- **polish**: F1, MCC. Credit risk prediction in the Polish market.
- **taiwan**: F1, MCC. Credit risk prediction in the Taiwanese market.
- **portoseguro**: F1, MCC. Claim analysis in the Brazilian market.
- **travelinsurance**: F1, MCC. Travel insurance claim prediction.
- **MultiFin-ES**: F1. Multi-class financial sentiment analysis in Spanish.
- **EFP**: F1. Financial phrase classification in Spanish.
- **EFPA**: F1. Financial argument classification in Spanish.
- **FinanceES**: F1. Financial sentiment classification in Spanish.
- **TSA-Spanish**: F1. Sentiment analysis in Spanish.


To ensure a fair and unbiased assessment of the models' true capabilities, all evaluations are conducted in zero-shot settings (0-shots). This approach eliminates any potential advantage from task-specific fine-tuning, providing a clear indication of how well the models can generalize to new tasks.

Given the nature of the tasks, which include multiple-choice and yes/no questions, we extract options from the generated text to evaluate performance.

Please, consider reaching out to us through the discussions tab if you are working on benchmarks for financial LLMs and willing to see them on this leaderboard as well. Your benchmark might change the whole game for financial models!

GPUs are provided by Wuhan University and the University of Florida for the evaluations.

## Details and Logs

- Detailed numerical results in the [results FinBen dataset](https://huggingface.co/datasets/FinBen/results)
- Community queries and running status in the [requests FinBen dataset](https://huggingface.co/datasets/FinBen/requests)

## More Resources

If you still have questions, you can check our github repository [here](https://github.com/The-FinAI/PIXIU).
"""

EVALUATION_QUEUE_TEXT = """
## Some good practices before submitting a model

### 1) Make sure you can load your model and tokenizer using AutoClasses:
```python
from transformers import AutoConfig, AutoModel, AutoTokenizer
config = AutoConfig.from_pretrained("your model name", revision=revision)
model = AutoModel.from_pretrained("your model name", revision=revision)
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
```
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.

Note: make sure your model is public!
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!

### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

### 3) Make sure your model has an open license!
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗

### 4) Fill up your model card
When we add extra information about models to the leaderboard, it will be automatically taken from the model card

## In case of model failure
If your model is displayed in the `FAILED` category, its execution stopped.
Make sure you have followed the above steps first.
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@misc{xie2024finben,
          title={The FinBen: An Holistic Financial Benchmark for Large Language Models},
          author={Qianqian Xie and Weiguang Han and Zhengyu Chen and Ruoyu Xiang and Xiao Zhang and Yueru He and Mengxi Xiao and Dong Li and Yongfu Dai and Duanyu Feng and Yijing Xu and Haoqiang Kang and Ziyan Kuang and Chenhan Yuan and Kailai Yang and Zheheng Luo and Tianlin Zhang and Zhiwei Liu and Guojun Xiong and Zhiyang Deng and Yuechen Jiang and Zhiyuan Yao and Haohang Li and Yangyang Yu and Gang Hu and Jiajia Huang and Xiao-Yang Liu and Alejandro Lopez-Lira and Benyou Wang and Yanzhao Lai and Hao Wang and Min Peng and Sophia Ananiadou and Jimin Huang},
          year={2024},
          eprint={2402.12659},
          archivePrefix={arXiv},
          primaryClass={cs.CL}
        }
"""