# coding=utf-8 # author: xusong # time: 2022/8/23 16:06 """ ## TODO: - http get方式获取参数，(高优先级) - i18 国际化 https://blog.csdn.net/qq_26212731/article/details/78457198 request.header中也有language - iter_vocab 的 warmup - add_special_token 开关 - theme 开关 light/dark - token_id/tokens/bytes 开关 - 通过 javascript 添加 hover_text - 给方法 + 缓存，避免重复调用 - 英文 utf-8编码 - 词典支持下载 - 中文字词统计，是否要包括 _ G 等字符 - baichuan的单字数量怎么两万多个？ - OOV - feedback位置 - gpt4, gpt3.5 的overlap tokens 有问题。 plots table ## related demo - [](http://text-processing.com/demo/tokenize/) - [gpt-tokenizer](https://gpt-tokenizer.dev/) - [llama-tokenizer-js](https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/) - [](https://huggingface.co/spaces/Xenova/the-tokenizer-playground) ## 可视化 [ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ] """ import gradio as gr from vocab import all_tokenizers from util import * # llama chatglm_6b gpt_nexo_20b baichuan baichuan_7b examples_zh = [ ["空格测试： 2个空格 8个空格", "llama", "chatglm_6b"], # chatglm 有blank_n, ["标点测试：，。！？；", "baichuan_7b", "llama"], ["符号测试：🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"], ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"], ["中文简体：宽带，繁体：樂來", "baichuan_7b", "llama"], ] examples = [ ["spaces: 2spaces 8spaces", "llama", "chatglm_6b"], # chatglm 有blank_n, ["punctuations: ,./?\"，。！？；", "baichuan_7b", "llama"], ["symbols: 🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"], ["digits: (10086 + 98) = 100184", "baichuan_7b", "llama"], ] # jieba.enable_parallel() # flask中没办法parallel def example_fn(example_idx): return examples[example_idx] """Replace this text in the input field to see how tokenization works """ default_user_input = """Replace this text in the input field to see how tokenization works 华为发布Mate60手机 ラグビーワールドカップ2023フランス""" default_tokenizer_type_1 = "llama" default_tokenizer_type_2 = "internlm_chat_7b" default_stats_vocab_size_1, default_stats_zh_token_size_1 = basic_count(default_tokenizer_type_1) default_stats_vocab_size_2, default_stats_zh_token_size_2 = basic_count(default_tokenizer_type_2) default_stats_overlap_token_size = get_overlap_token_size(default_tokenizer_type_1, default_tokenizer_type_2)[0] default_output_text_1, default_output_table_1, default_output_len_1 = tokenize(default_user_input, default_tokenizer_type_1, update=False) default_output_text_2, default_output_table_2, default_output_len_2 = tokenize(default_user_input, default_tokenizer_type_2, update=False) with gr.Blocks(css="style.css") as demo: gr.HTML("""

Tokenizer Arena ⚔️

""") # links: https://www.coderstool.com/utf8-encoding-decoding # 功能：输入文本，进行分词 # 分词器：常见的分词器有集中， # 背景：方便分词、看词粒度、对比 # # Byte: 表示分词 with gr.Row(): gr.Markdown("## Input Text") dropdown_examples = gr.Dropdown( # ["空格测试", "标点测试", "符号测试", "数字测试"], ["spaces", "punctuations", "symbols", "digits"], value="Examples", type="index", show_label=False, container=False, scale=0, elem_classes="example-style" ) user_input = gr.Textbox( value=default_user_input, label="Input Text", lines=5, show_label=False, ) # placeholder="Enter sentence here..." # gr.Examples( # examples, # None, # ) gr.Markdown("## Tokenization") with gr.Row(): with gr.Column(scale=6): with gr.Group(): tokenizer_type_1 = gr.Dropdown( all_tokenizers, value=default_tokenizer_type_1, label="Tokenizer 1", ) with gr.Group(): """
69
Characters
""" with gr.Row(): stats_vocab_size_1 = gr.TextArea( value=default_stats_vocab_size_1, label="VocabSize", lines=1, elem_classes="statistics" ) stats_zh_token_size_1 = gr.TextArea( value=default_stats_zh_token_size_1, label="ZH char/word", lines=1, elem_classes="statistics" ) stats_overlap_token_size_1 = gr.TextArea( value=default_stats_overlap_token_size, label="Overlap Tokens", lines=1, elem_classes="statistics" ) # stats_3 = gr.TextArea( # label="Compress Rate", # lines=1, # elem_classes="statistics" # ) # https://www.onlinewebfonts.com/icon/418591 gr.Image("images/VS.svg", scale=1, show_label=False, show_download_button=False, container=False, show_share_button=False) with gr.Column(scale=6): with gr.Group(): tokenizer_type_2 = gr.Dropdown( all_tokenizers, value=default_tokenizer_type_2, label="Tokenizer 2", ) with gr.Group(): with gr.Row(): stats_vocab_size_2 = gr.TextArea( value=default_stats_vocab_size_2, label="VocabSize", lines=1, elem_classes="statistics" ) stats_zh_token_size_2 = gr.TextArea( value=default_stats_zh_token_size_2, label="ZH char/word", # 中文字/词 lines=1, elem_classes="statistics" ) # stats_6 = gr.TextArea( # label="Compress Rate", # lines=1, # elem_classes="statistics" # ) stats_overlap_token_size_2 = gr.TextArea( value=default_stats_overlap_token_size, label="Overlap Tokens", lines=1, elem_classes="statistics" ) # TODO: 图 表 压缩率 with gr.Row(): with gr.Column(): output_text_1 = gr.Highlightedtext( value=default_output_text_1, label=f"Tokens: {default_output_len_1}", show_legend=True, elem_classes="space-show" ) with gr.Column(): output_text_2 = gr.Highlightedtext( value=default_output_text_2, label=f"Tokens: {default_output_len_2}", show_legend=True, elem_classes="space-show" ) with gr.Row(): output_table_1 = gr.Dataframe( value=default_output_table_1, headers=["TokenID", "Byte", "Text"], datatype=["str", "str", "str"], # elem_classes="space-show", # 给整个Dataframe加这个css不起作用，因此直接修改cell-wrap ) output_table_2 = gr.Dataframe( value=default_output_table_2, headers=["TokenID", "Token", "Text"], datatype=["str", "str", "str"], ) tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1]) # 下面两个好像可以合并 tokenizer_type_1.change(basic_count, [tokenizer_type_1], [stats_vocab_size_1, stats_zh_token_size_1]) tokenizer_type_1.change(get_overlap_token_size, [tokenizer_type_1, tokenizer_type_2], [stats_overlap_token_size_1, stats_overlap_token_size_2]) user_input.change(tokenize_pair, [user_input, tokenizer_type_1, tokenizer_type_2], [output_text_1, output_table_1, output_text_2, output_table_2]) # , pass_request=1 tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2]) tokenizer_type_2.change(basic_count, [tokenizer_type_2], [stats_vocab_size_2, stats_zh_token_size_2]) tokenizer_type_2.change(get_overlap_token_size, [tokenizer_type_1, tokenizer_type_2], [stats_overlap_token_size_1, stats_overlap_token_size_2]) dropdown_examples.change( example_fn, dropdown_examples, [user_input, tokenizer_type_1, tokenizer_type_2] ) # start up 初始化 # user_input.update(user_input.value + "___") if __name__ == "__main__": demo.queue(max_size=20).launch() # demo.launch()