wait for soft plotly

app.py

@@ -60,10 +60,13 @@ def get_text(ids_list=[0.1,0.7], tokenizer=None):
 #     # 这里只是简单地返回 IDs 和损失值，但是可以根据实际需求添加颜色或其他样式
 #     return [1, 2], [0.1, 0.7]
 
-def harness_eval(question, choices, answer_index, model=None,tokenizer=None):
+def harness_eval(question,  answer_index, answer_type, model=None,*choices,):
     '''
     use harness to test one question,  can specify the model, (extract or ppl)
     '''
+    # print(f'question,choices,answer_index,model,tokenizer: {question,choices,answer_index,model,tokenizer}')
+    print(f'type of choices: {type(choices)} and type of choices[0]: {type(choices[0])}')
+    print(f'choices: {choices}')
     # TODO add the model and its score
     # torch.nn.functional.softmax(output.logits, dim=0)
     # topk = torch.topk(output.logits, 5)
@@ -73,7 +76,7 @@ def harness_eval(question, choices, answer_index, model=None,tokenizer=None):
 
 
 
-def plotly_plot():#(df, x, y, color,title, x_title, y_title):
+def plotly_plot_text():#(df, x, y, color,title, x_title, y_title):
         # plotly_plot(sample_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl')
     df=pd.read_csv('./data/tmp.csv')
     df['date'] = pd.to_datetime(df['date'])
@@ -91,6 +94,24 @@ def plotly_plot():#(df, x, y, color,title, x_title, y_title):
     # fig.update_layout()
     return fig
 
+def plotly_plot_question():#(df, x, y, color,title, x_title, y_title):
+        # plotly_plot(sample_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl')
+    df=pd.read_csv('./data/meta_gjo_df.csv')
+    df['date'] = pd.to_datetime(df['End Time'])
+    # sort by date
+    df.sort_values(by='date', inplace=True)
+
+    # use a dic to filter the dataframe
+    # df = df[df['file_name'] == 'arxiv_computer_science']
+
+    x,y,color,title, x_title, y_title='date', 'Right Possibility', 'model','Right Possibility with time', 'time', 'Right Possibility'
+
+    fig = px.line(df, x=x, y=y, color=color,title=title)
+    fig.update_xaxes(title_text=x_title)
+    fig.update_yaxes(title_text=y_title)
+    # fig.update_layout()
+    return fig
+
 # def plotly_plot(df, x, y, color, title, x_title, y_title):
 #     fig = px.line(df, x=x, y=y, color=color, title=title)
 #     fig.update_xaxes(title_text=x_title)
@@ -165,9 +186,14 @@ def color_pipeline(texts=["Hi","FreshEval","!"],  model=None):
 
 
 # TODO can this be global ? maybe need session to store info of the user
+# visible_btn_num = 4
 
 # 创建 Gradio 界面
 with gr.Blocks() as demo:
+    # visible_btn_num = 4
+    model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here
+
+
     with gr.Tab("color your text"):
         with gr.Row():
             text_input = gr.Textbox(label="input text", placeholder="input your text here...")
@@ -175,7 +201,7 @@ with gr.Blocks() as demo:
             # TODO craw and drop the file
 
             # loss_input = gr.Number(label="loss")
-            model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here
+            # model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here
         output_box=gr.HighlightedText(label="colored text")#,interactive=True
         
         gr.Examples(
@@ -219,7 +245,46 @@ with gr.Blocks() as demo:
         '''
         question=gr.Textbox(label="input question", placeholder='input your question here...')
         answer_index=gr.Textbox(label="right answer index", placeholder='index for right anser here, start with 0')#TODO add multiple choices, 
-        choices=gr.Textbox(placeholder='input your other choices here...')
+        # model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here
+
+        btn_list = []
+
+        # choices=gr.Textbox(placeholder='input your other choices here...')
+        
+        button_limit=10
+        # global visible_btn_num
+        visible_btn_num = 4
+
+        from gradio_samples.add_components import add_one_btn, remove_one_btn, get_text_content
+
+        # use partial function
+        from functools import partial
+        add_one_btn=partial(add_one_btn,button_limit=button_limit,)#visible_btn_num = visible_btn_num)
+        remove_one_btn=partial(remove_one_btn,button_limit=button_limit,)#visible_btn_num = visible_btn_num)
+            
+        # with gr.Blocks() as demo:
+        with gr.Row():
+            for i in range(button_limit):
+                if i<visible_btn_num:
+                    btn = gr.Textbox(visible=True)
+                else:
+                    btn = gr.Textbox(visible=False)
+                btn_list.append(btn)
+        b = gr.Button("add_one_choice(make sure every existing choice is filled)")
+        print(f'len(btn_list): {len(btn_list)}')
+        b.click(add_one_btn, btn_list, btn_list)
+        b = gr.Button("remove_one_choice")
+        b.click(remove_one_btn, btn_list, btn_list)
+
+        # # print(f'len(btn_list): {len(btn_list)}')
+
+        # print('btn_list is ',type(btn_list),btn_list)
+
+
+        # b = gr.Button("Get Text Content")
+        # output = gr.Textbox()
+        # b.click(get_text_content, btn_list, output)
+
         
         # test_button=gr.Button('test').click(harness_eval())# TODO figure out the input and output
 
@@ -227,15 +292,20 @@ with gr.Blocks() as demo:
         #TODO add the model and its score
         answer_label=gr.Label('the answers\'s detail')# RETURN the answer and its score,in the form of dic{str: float}
 
-        test_question_button=gr.Button('test question').click(harness_eval,inputs=[question, choices, answer_index ,answer_type],outputs=[answer_label])
+        test_question_button=gr.Button('test question').click(harness_eval,inputs=[question, answer_index ,model_input,answer_type,*btn_list],outputs=[answer_label])
 
         forecast_q='A Ukrainian counteroffensive began in 2023, though territorial gains by November 2023 were limited (Economist, BBC, Newsweek). The question will be suspended on 31 July 2024 and the outcome determined using data as reported in the Brookings Institution\'s "Ukraine Index" (Brookings Institution - Ukraine Index, see "Percentage of Ukraine held by Russia" chart). If there is a discrepancy between the chart data and the downloaded data (see "Get the data" within the "NET TERRITORIAL GAINS" chart border), the downloaded data will be used for resolution.'
-        answer_list=['Less than 5%','At least 5%, but less than 10%','At least 10%, but less than 15%','At least 15%, but less than 20%','20% or more' ]
+        # answer_list=['Less than 5%','At least 5%, but less than 10%','At least 10%, but less than 15%','At least 15%, but less than 20%','20% or more' ]
+        answer_list=['Less than 5%','At least 5%, but less than 10%','At least 10%, but less than 15%','15% or more' ]
          
+        # gr.Examples([
+        #             [forecast_q, '&&&&&&'.join(answer_list), '0']
+        #              ],
+        #             [question, choices, answer_index])
         gr.Examples([
-                    [forecast_q, '&&&&&&'.join(answer_list), '0']
-                     ],
-                    [question, choices, answer_index])
+            [forecast_q, answer_list[0],answer_list[1],answer_list[2],answer_list[3], '0']
+                ],
+            [question,btn_list[0],btn_list[1],btn_list[2],btn_list[3], answer_index])
 
         date_time_input = gr.Textbox(label="the date when the text is generated")#TODO add date time input
         description_input = gr.Textbox(label="description of the text")
@@ -259,20 +329,20 @@ with gr.Blocks() as demo:
         # load the json file with time,
 
         # sample_df=pd.DataFrame({'time':pd.date_range('2021-01-01', periods=6), 'ppl': [1,2,3,4,5,6]})
-        pd_df=pd.read_csv('./data/tmp.csv')
-        pd_df['date'] = pd.to_datetime(pd_df['date'])
-        print(pd_df.head)
-        # gr_df=gr.Dataframe(pd_df)
-        gr_df=pd_df
+        # pd_df=pd.read_csv('./data/tmp.csv')
+        # pd_df['date'] = pd.to_datetime(pd_df['date'])
+        # print(pd_df.head)
+        # # gr_df=gr.Dataframe(pd_df)
+        # gr_df=pd_df
 
 
         # print(gr_df.head)
-        print('done')
+        # print('done')
         # sample
         plot=gr.Plot(label='model text ppl')
         # plotly_plot(gr_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl')
         # draw_pic_button=gr.Button('draw the pic').click(plotly_plot,inputs=['gr_df', 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl'],outputs=[plot])
-        draw_pic_button=gr.Button('draw the pic').click(plotly_plot,inputs=[],outputs=[plot])
+        draw_pic_button=gr.Button('draw the pic').click(plotly_plot_text,inputs=[],outputs=[plot])
 
 
     
@@ -280,7 +350,23 @@ with gr.Blocks() as demo:
         '''
         see the matplotlib example, to see ppl with time, select the models
         ''' 
-        #
+
+        # pd_df=pd.read_csv('./data/meta_gjo_df.csv')
+        # pd_df['date'] = pd.to_datetime(pd_df['end_date'])
+        # print(pd_df.head)
+        # gr_df=gr.Dataframe(pd_df)
+        # gr_df=pd_df
+
+
+        # print(gr_df.head)
+        # print('done')
+        # sample
+        plot=gr.Plot(label='question acc with time')
+        # plotly_plot(gr_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl')
+        # draw_pic_button=gr.Button('draw the pic').click(plotly_plot,inputs=['gr_df', 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl'],outputs=[plot])
+        draw_pic_button=gr.Button('draw the pic').click(plotly_plot_question,inputs=[],outputs=[plot])
+
+
 
 
     with gr.Tab("hot questions"):
@@ -295,50 +381,3 @@ with gr.Blocks() as demo:
 
 demo.launch(debug=True)
 
-
-
-
-
-# import gradio as gr
-# import os
-# os.system('python -m spacy download en_core_web_sm')
-# import spacy
-# from spacy import displacy
-
-# nlp = spacy.load("en_core_web_sm")
-
-# def text_analysis(text):
-#     doc = nlp(text)
-#     html = displacy.render(doc, style="dep", page=True)
-#     html = (
-#         "<div style='max-width:100%; max-height:360px; overflow:auto'>"
-#         + html
-#         + "</div>"
-#     )
-#     pos_count = {
-#         "char_count": len(text),
-#         "token_count": 0,
-#     }
-#     pos_tokens = []
-
-#     for token in doc:
-#         pos_tokens.extend([(token.text, token.pos_), (" ", None)])
-
-#     return pos_tokens, pos_count, html
-
-# demo = gr.Interface(
-#     text_analysis,
-#     gr.Textbox(placeholder="Enter sentence here..."),
-#     ["highlight", "json", "html"],
-#     examples=[
-#         ["What a beautiful morning for a walk!"],
-#         ["It was the best of times, it was the worst of times."],
-#     ],
-# )
-
-# demo.launch()
-
-
-
-# # lm-eval 
-# # lm-evaluation-harness 

get_loss/get_loss_hf.py

@@ -128,7 +128,8 @@ def load_hf_model(path, cache_path):
         hf_model = AutoModelForCausalLM.from_pretrained(path,
                                                         device_map=device,
                                                         trust_remote_code=True,
-                                                        cache_dir=cache_path).eval()
+                                                        cache_dir=cache_path,
+                                                        output_attentions=True).eval()
         # output_attentions=True)
     else:
         hf_model = AutoModelForCausalLM.from_pretrained(path,

gradio_samples/add_components.py

@@ -32,37 +32,91 @@
 
 import gradio as gr
 
-def words():
-    sentence = "A test of Gradio"
-    words = sentence.split()
+
+
+def add_one_btn(*btn_list,button_limit=10):
+    print(f'input is {btn_list},type is {type(btn_list)},len is {len(btn_list)},type[0] is {type(btn_list[0])}')
+    # global visible_btn_num
+    # global button_limit
+    # visible_btn_num = sum([1 for btn in btn_list if btn.visible])
+    visible_btn_num = sum([1 for btn in btn_list if len(btn)>0])
+    print(f'visible_btn_num: {visible_btn_num}')
+
+
+    if visible_btn_num<button_limit:
+        visible_btn_num += 1
+    else:
+        print(f'button limit {button_limit} reached')
+    print(f'len(btn_list): {len(btn_list)}')
+    words = btn_list[:visible_btn_num]#sentence.split()
     update_show = [gr.Textbox(visible=True, value=w,interactive=True) for w in words]
-    update_hide = [gr.Textbox(visible=False, value="",interactive=True) for _ in range(10-len(words))]
-    return update_show + update_hide 
+    update_hide = [gr.Textbox(visible=False, value="",interactive=True) for _ in range(button_limit-len(words))]
+    # update_hide = [gr.Textbox(visible=False, value="",interactive=True) for _ in range(10-len(words))]
+    rtn_list=update_show + update_hide 
+    # print(f'len(rtn_list): {len(rtn_list)}')
+    return rtn_list
+
+def remove_one_btn(*btn_list,button_limit=10):
+    # global visible_btn_num
+    # count the visible btn in the btn_list
+    # visible_btn_num = sum([1 for btn in btn_list if btn.visible])
+    visible_btn_num = sum([1 for btn in btn_list if len(btn)>0])
+    print(f'visible_btn_num: {visible_btn_num}')
+
+    # global button_limit
+    if visible_btn_num>0:
+        visible_btn_num -= 1
+    else:
+        print(f'button limit {button_limit} reached')
+    # print(f'len(btn_list): {len(btn_list)}')
+    words = btn_list[:visible_btn_num]#sentence.split()
+    update_show = [gr.Textbox(visible=True, value=w,interactive=True) for w in words]
+    update_hide = [gr.Textbox(visible=False, value="",interactive=True)for _ in range(button_limit-len(words))]
+    # update_hide = [gr.Textbox(visible=False, value="",interactive=True) for _ in range(10-len(words))]
+    rtn_list=update_show + update_hide 
+    print(f'len(rtn_list): {len(rtn_list)}')
+    return rtn_list
 
     
 def get_text_content(*btn_list):
-    # make all the input as a list
-    # merge_list 
-    # rtn =[w.value for w in btn_list if w.visible]
-
+    print(f'input is {btn_list}')
     rtn =' '.join([w for w in btn_list ])
 
     print(rtn)
     return  rtn
 
 
-btn_list = []
-    
-with gr.Blocks() as demo:
-    with gr.Row():
-        for i in range(10):
-            btn = gr.Textbox(visible=False)
-            btn_list.append(btn)
-    b = gr.Button("Run")
-    b.click(words, None, btn_list)
-
-    b = gr.Button("Get Text Content")
-    output = gr.Textbox()
-    b.click(get_text_content, btn_list, output)
+
+
+if __name__ == '__main__':
+    button_limit=10
+    visible_btn_num = 4
+    btn_list = []
+    from functools import partial
+    add_one_btn_partial = partial(add_one_btn, button_limit=button_limit)#, visible_btn_num=visible_btn_num)
+    remove_one_btn_partial = partial(remove_one_btn, button_limit=button_limit)#, visible_btn_num=visible_btn_num)
         
-demo.launch(debug=True)
+    with gr.Blocks() as demo:
+        with gr.Row():
+            for i in range(button_limit):
+                if i<visible_btn_num:
+                    btn = gr.Textbox(visible=True)
+                else:
+                    btn = gr.Textbox(visible=False)
+                btn_list.append(btn)
+        b = gr.Button("add_one_choice")
+        print(f'len(btn_list): {len(btn_list)}')
+        b.click(add_one_btn_partial, btn_list, btn_list)
+        b = gr.Button("remove_one_choice")
+        b.click(remove_one_btn_partial, btn_list, btn_list)
+
+        # print(f'len(btn_list): {len(btn_list)}')
+
+        print(btn_list)
+
+
+        b = gr.Button("Get Text Content")
+        output = gr.Textbox()
+        b.click(get_text_content, btn_list, output)
+            
+    demo.launch(debug=True)

gradio_samples/draft.py

@@ -0,0 +1,41 @@
+        '''
+        use extract, or use ppl
+        '''
+        question=gr.Textbox(label="input question", placeholder='input your question here...')
+        answer_index=gr.Textbox(label="right answer index", placeholder='index for right anser here, start with 0')#TODO add multiple choices, 
+        # choices=gr.Textbox(placeholder='input your other choices here...')
+        
+        btn_list=[]
+        button_limit=10
+        with gr.Row():
+            for i in range(button_limit):
+                btn = gr.Textbox(visible=True)
+                # btn = gr.Textbox(visible=False)
+                btn_list.append(btn)
+        b = gr.Button("add_one_choice")
+
+        print(f'len(btn_list): {len(btn_list)},btn_list:{btn_list}')
+
+        
+        # test_button=gr.Button('test').click(harness_eval())# TODO figure out the input and output
+
+        answer_type=gr.Dropdown(label="answer type", choices=['extract', 'ppl'])
+        #TODO add the model and its score
+        answer_label=gr.Label('the answers\'s detail')# RETURN the answer and its score,in the form of dic{str: float}
+
+        # test_question_button=gr.Button('test question').click(harness_eval,inputs=[question, '&&&&&&'.join(btn_list), answer_index ,answer_type],outputs=[answer_label])
+        test_question_button=gr.Button('test question').click(harness_eval,inputs=[question, *btn_list, answer_index ,answer_type],outputs=[answer_label])
+
+        forecast_q='A Ukrainian counteroffensive began in 2023, though territorial gains by November 2023 were limited (Economist, BBC, Newsweek). The question will be suspended on 31 July 2024 and the outcome determined using data as reported in the Brookings Institution\'s "Ukraine Index" (Brookings Institution - Ukraine Index, see "Percentage of Ukraine held by Russia" chart). If there is a discrepancy between the chart data and the downloaded data (see "Get the data" within the "NET TERRITORIAL GAINS" chart border), the downloaded data will be used for resolution.'
+        answer_list=['Less than 5%','At least 5%, but less than 10%','At least 10%, but less than 15%','At least 15%, but less than 20%','20% or more' ]
+         
+        gr.Examples([
+                    [forecast_q, '0']
+                     ],
+                    [question, answer_index])
+
+        date_time_input = gr.Textbox(label="the date when the text is generated")#TODO add date time input
+        description_input = gr.Textbox(label="description of the text")
+        submit_button = gr.Button("submit a post or record").click()
+
+        #TODO add the model and its score