Login feature, token estimates

app.py

@@ -28,7 +28,7 @@ def save_json(score1, score2):
             json.dump({"relevance": score1, "novelty": score2, "datetime": datetime.now().isoformat()}, f)
             f.write("\n")
 
-# READING EXISTING DATA: this is used to read questionss
+# READING EXISTING DATA: this is used to read questions
 
 """
 from datasets import load_dataset
@@ -52,12 +52,19 @@ passage_texts = ["The IDEA also requires states receiving federal funds to educa
             "To assure that children attending public schools obtain a high quality education, the legislature shall make adequate provision to ensure that ... there are a sufficient number of classrooms ....",
             "courts should not disturb a state's denial of IDEA reimbursement where the chief benefits of the chosen school are the kind of advantages that might be preferred by parents of any child, disabled or not. Rather, the unilateral private placement is only appropriate if it provides education instruction [specially ] designed to meet the unique needs of a handicapped child.",
             "It is also likely that residents of the States will attend out-of-state schools that invoke the Exemptions, and that such students will seek contraceptive services through programs in their home states, also giving rise to fiscal injuries to the States that only a nationwide injunction can remedy.",
-            "Although state classifications based on alienage are generally suspect, a state may reserve a government position for citizens if it is related to self-governance, involves policymaking, or requires exercise of important discretionary power over citizens. In these cases, only a rationality test is used. A public school teacher at the primary and secondary school level performs an important governmental function (e.g., he influences students' attitudes about government, the political process, citizenship, etc.), and therefore the exclusion of aliens is rationally related to the state's interest in furthering educational goals. [Ambach v. Norwick (1979)]"
-            ]
+            "Although state classifications based on alienage are generally suspect, a state may reserve a government position for citizens if it is related to self-governance, involves policymaking, or requires exercise of important discretionary power over citizens. In these cases, only a rationality test is used. A public school teacher at the primary and secondary school level performs an important governmental function (e.g., he influences students' attitudes about government, the political process, citizenship, etc.), and therefore the exclusion of aliens is rationally related to the state's interest in furthering educational goals. [Ambach v. Norwick (1979)]",
+            "Passage 9",
+            "Passage 10"]
+generation_4 = "Here is a generation at 4 passages."
+generation_6 = "Here is a generation at 6 passages."
+generation_10 = "Here is a generation at 10 passages."
+gold_passage = "GOLD PASSAGE"
 step = 0
 
 # BLOCKS: main user interface
 
+user_id = "NO_ID"
+
 with gr.Blocks() as user_eval:
     # Title text introducing study
     gr.Markdown("""
@@ -115,25 +122,44 @@ with gr.Blocks() as user_eval:
 
     # Question and answering dynamics
     with gr.Row() as question:
-        with gr.Column():
-            gr.Markdown("---")
-            gr.Markdown("**Question**")
-            gr.Markdown(question_text)
-            a = gr.Button(answers_text[0])
-            b = gr.Button(answers_text[1])
-            c = gr.Button(answers_text[2])
-            d = gr.Button(answers_text[3])
-
-            def answer():
-                return {
-                    question: gr.Row(visible = False),
-                    evals: gr.Row(visible = True)
-                }
-            
-            a.click(fn = answer, outputs = [question, evals])
-            b.click(fn = answer, outputs = [question, evals])
-            c.click(fn = answer, outputs = [question, evals])
-            d.click(fn = answer, outputs = [question, evals])
+        if user_id == "NO_ID":
+            with gr.Column():
+                gr.Markdown("---")
+                gr.Markdown("# Enter email to start")
+                gr.Markdown("Thank you so much for your participation in our study! Please enter your email — we're using it to keep track of which questions you've answered and which you haven't seen. Use the same email every time to keep your progress saved. :)")
+                email = gr.Textbox("Email")
+                s = gr.Button("Start!")
+
+                def submit_email():
+                    global user_id
+                    user_id = email
+                    return {
+                        question: gr.Row(visible = True),
+                    }
+                
+                s.click(fn = submit_email, outputs = [question])
+        else:
+            with gr.Column():
+                gr.Markdown("---")
+                gr.Markdown("**Question**")
+                gr.Markdown(question_text)
+                a = gr.Button(answers_text[0])
+                b = gr.Button(answers_text[1])
+                c = gr.Button(answers_text[2])
+                d = gr.Button(answers_text[3])
+
+                def answer():
+                    return {
+                        question: gr.Row(visible = False),
+                        evals: gr.Row(visible = True)
+                    }
+                
+                a.click(fn = answer, outputs = [question, evals])
+                b.click(fn = answer, outputs = [question, evals])
+                c.click(fn = answer, outputs = [question, evals])
+                d.click(fn = answer, outputs = [question, evals])
 
 # Starts on question, switches to evaluation after the user answers
-user_eval.launch()
+user_eval.launch()
+
+# https://github.com/gradio-app/gradio/issues/5791

token_estimates.py

@@ -0,0 +1,37 @@
+import csv 
+import tiktoken
+
+questions_and_passages = []
+
+with open("sample.csv") as f: 
+    reader = csv.reader(f)
+    next(f)
+    for row in reader:
+        entry = []
+        entry.append((row[1] + " " + row[2]).strip()) # optional prompt + question
+        entry.append(row[9]) # first of 10 passages
+        for i in range(10, 19): entry[1] += " " + row[i] # next 9 passages all separated with a space
+        questions_and_passages.append(entry)
+
+enc = tiktoken.encoding_for_model("gpt-4o")
+
+question_tokens = 0
+question_passage_tokens = 0
+max_qt = 0
+max_pt = 0
+max_qpt = 0
+
+for entry in questions_and_passages:
+    qt = len(enc.encode(entry[0]))
+    question_tokens += qt 
+    if qt > max_qt: max_qt = qt
+    pt = len(enc.encode(entry[1]))
+    question_passage_tokens += qt + pt
+    if pt > max_pt: max_pt = pt
+    if qt + pt > max_qpt: max_qpt = qt + pt
+
+print("Average question length, gpt-4o tokens: " + str(question_tokens / len(questions_and_passages)))
+print("Longest question (tokens): " + str(max_qt))
+print("Average question + 10 passages length, gpt-4o tokens: " + str(question_passage_tokens / len(questions_and_passages)))
+print("Longest set of ten passages (tokens): " + str(max_pt))
+print("Longest combination of question and passages: " + str(max_qpt))