update

compression_app.py

@@ -27,7 +27,8 @@ from compression_util import get_compression_leaderboard, common_corpuses
 # exactly reconstructed from compressed tokens
 docs = """## 📖 What is a good tokenizer?
 
-From a compression perspective, a good tokenizer should be lossless, and keep high compression rate (less tokens).
+From a compression perspective, a good tokenizer should be lossless,
+and keep high compression rate (less tokens for a given text).
 The encoding and decoding process can be formulated as
 ```python
     token_ids = tokenizer.encode(input_text)    # compressed tokens
@@ -142,9 +143,9 @@ with gr.Blocks(theme=theme) as demo:
             )
 
     gr.Markdown("## 🏆 Compression Rate Leaderboard\n"
-                "The leaderboard aims to evaluate tokenizer performance on different languages.\n"
+                "This leaderboard aims to evaluate tokenizer performance on different languages.\n"
                 "Lower `oov_ratio` refers to less out-of-vocabulary tokens.\n"
-                "Higher `char/token` means less words be segmented into subwords."
+                "Lower `char/token` means more words might be segmented into subwords."
                 )
     search_bar = gr.Textbox(
         placeholder="🔍 Search by tokenizer or organization (e.g., 'llama', 'openai') and press ENTER...",

compression_util.py

@@ -297,9 +297,7 @@ def get_compression_leaderboard(
         reverse_unit = f"{file_size_unit}/{token_number_unit}"
         stats = to_dataframe(stats, ["char/token", unit, reverse_unit])
         stats = stats.sort_values(["oov_ratio", "char/token"], ascending=[True, False])
-
         # stats = stats.sort_values(["oov_ratio", unit], ascending=[True, True])
-
         stats = stats.rename(columns={"oov_ratio": f' ⬆️oov_ratio'}).rename(columns={"char/token": ' ⬇️char/token'})  #
     return stats