Upload 9 files

app.py

@@ -8,28 +8,17 @@ from transformers import (
     pipeline,
 )
 import spacy
-# import pandas as pd
-from lib.utils import ContextRetriever
-
-
-#### TO DO:######
-# build out functions for:
-#     * formatting input into document retrieval query (spaCy)
-#     * document retrieval based on query (wikipedia library)
-#     * document postprocessing into passages
-#     * ranking passage based on BM25 scores for query (rank_bm25)
-#     * feeding passages into RoBERTa an reporting answer(s) and passages as evidence
-# decide what to do with examples
-
-### CAN REMOVE:#####
-    # * context collection
-    # *
+from lib.utils import (
+    ContextRetriever,
+    get_examples,
+    generate_query,
+    generate_answer,
+)
 
-########################
-### Helper functions ###
-########################
+#################################
+### Model retrieval functions ###
+#################################
 
-# Build trainer using model and tokenizer from Hugging Face repo
 @st.cache_resource(show_spinner=False)
 def get_pipeline():
     """
@@ -70,57 +59,6 @@ def get_spacy():
     )
     return nlp
 
-def generate_query(nlp,text):
-    """
-    Process text into a search query,
-    only retaining nouns, proper nouns,
-    numerals, verbs, and adjectives
-    Parameters:
-    -----------
-    nlp : spacy.Pipe
-        spaCy pipeline for processing search query
-    text : str
-        The input text to be processed
-    Returns:
-    --------
-    query : str
-        The condensed search query
-    """
-    tokens = nlp(text)
-    keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
-    query = ' '.join(token.text for token in tokens \
-                         if token.pos_ in keep)
-    return query
-
-def fill_in_example(i):
-    """
-    Function for context-question example button click
-    """
-    st.session_state['response'] = ''
-    st.session_state['question'] = ex_q[i]
-
-def clear_boxes():
-    """
-    Function for field clear button click
-    """
-    st.session_state['response'] = ''
-    st.session_state['question'] = ''
-
-# def get_examples():
-#     """
-#     Retrieve pre-made examples from a .csv file
-#     Parameters: None
-#     -----------
-#     Returns:
-#     --------
-#     questions, contexts : list, list
-#         Lists of examples of corresponding question-context pairs
-        
-#     """
-#     examples = pd.read_csv('examples.csv')
-#     questions = list(examples['question'])
-#     return questions
-
 #############
 ### Setup ###
 #############
@@ -134,40 +72,71 @@ else:
     device = "cpu"
 
 # Initialize session state variables
-if 'response' not in st.session_state:
-    st.session_state['response'] = ''
-if 'question' not in st.session_state:
-    st.session_state['question'] = ''
-
-# Retrieve trained RoBERTa pipeline for Q&A
-# and spaCy pipeline for processing search query
+for tab in ['basic','semi','auto']:
+    if tab not in st.session_state:
+        st.session_state[tab] = {}
+    for field in ['question','context','query','response']:
+        if field not in st.session_state[tab]:
+            st.session_state[tab][field] = ''
+for field in ['page_options','selected_pages']:
+    if field not in st.session_state['semi']:
+        st.session_state['semi'][field] = []
+        
+# Retrieve models
 with st.spinner('Loading the model...'):
     qa_pipeline = get_pipeline()
     nlp = get_spacy()
 
-# # Grab example question-context pairs from csv file
-# ex_q, ex_c = get_examples()
+# Retrieve example questions and contexts
+examples = get_examples()
+# ex_queries, ex_questions, ex_contexts = get_examples()
+if 'ex_questions' not in st.session_state['semi']:
+    st.session_state['semi']['ex_questions'] = len(examples[1][0])*['']
+    
+################################
+### Initialize App Structure ###
+################################
 
-###################
-### App content ###
-###################
+tabs = st.tabs([
+    'RoBERTa Q&A model',
+    'Basic extractive Q&A',
+    'User-guided Wiki Q&A',
+    'Automated Wiki Q&A',
+])
 
-# Intro text
-st.header('RoBERTa answer retieval')
-st.markdown('''
-This app demonstrates the answer-retrieval capabilities of a fine-tuned RoBERTa (Robustly optimized Bidirectional Encoder Representations from Transformers) model.
+with tabs[0]:
+    intro_container = st.container()
+with tabs[1]:
+    basic_title_container = st.container()
+    basic_example_container = st.container()
+    basic_input_container = st.container()
+    basic_response_container = st.container()
+with tabs[2]:
+    semi_title_container = st.container()
+    semi_query_container = st.container()
+    semi_page_container = st.container()
+    semi_input_container = st.container()
+    semi_response_container = st.container()
+with tabs[3]:
+    auto_title_container = st.container()
+    auto_example_container = st.container()
+    auto_input_container = st.container()
+    auto_response_container = st.container()
 
-Please type in a question and click submit.  When you do, a few things will happen:
-1. A Wikipedia search will be performed based on your question
-2. Candidate passages will be ranked based on a similarity score as compared to your question
-3. RoBERTa will search the best candidate passages to find the answer to your question
+##############################
+### Populate tab - Welcome ###
+##############################
 
-If the model cannot find the answer to your question, it will tell you so.
-''')
-with st.expander('Click to read more about the model...'):
+with intro_container:
+    # Intro text
+    st.header('RoBERTa Q&A model')
     st.markdown('''
+    This app demonstrates the answer-retrieval capabilities of a fine-tuned RoBERTa (Robustly optimized Bidirectional Encoder Representations from Transformers) model.
+    ''')
+    with st.expander('Click to read more about the model...'):
+        st.markdown('''
 * [Click here](https://huggingface.co/etweedy/roberta-base-squad-v2) to visit the Hugging Face model card for this fine-tuned model.
-* To create this model, the [RoBERTa base model](https://huggingface.co/roberta-base) was fine-tuned on Version 2 of [SQuAD (Stanford Question Answering Dataset)](https://huggingface.co/datasets/squad_v2), a dataset of context-question-answer triples.
+* To create this model, I fine-tuned the [RoBERTa base model](https://huggingface.co/roberta-base) Version 2 of [SQuAD (Stanford Question Answering Dataset)](https://huggingface.co/datasets/squad_v2), a dataset of context-question-answer triples.
 * The objective of the model is "extractive question answering", the task of retrieving the answer to the question from a given context text corpus.
 * SQuAD Version 2 incorporates the 100,000 samples from Version 1.1, along with 50,000 'unanswerable' questions, i.e. samples in the question cannot be answered using the context given.
 * The original base RoBERTa model was introduced in [this paper](https://arxiv.org/abs/1907.11692) and [this repository](https://github.com/facebookresearch/fairseq/tree/main/examples/roberta).  Here's a citation for that base model:
@@ -195,102 +164,320 @@ with st.expander('Click to read more about the model...'):
   bibsource = {dblp computer science bibliography, https://dblp.org}
 }
 ```
-''')
-# st.markdown('''
-# Please type or paste a context paragraph and question you'd like to ask about it.  The model will attempt to answer the question based on the context you provided.  If the model cannot find the answer in the context, it will tell you so - the model is also trained to recognize when the context doesn't provide the answer.
-
-# Your results will appear below the question field when the model is finished running.
+    ''')
+    st.markdown('''
+Use the menu on the left side to navigate between different app components:
+1. A basic Q&A tool which allows the user to ask the model to search a user-provided context paragraph for the answer to a user-provided question.
+2. A user-guided Wiki Q&A tool which allows the user to search for one or more Wikipedia pages and ask the model to search those pages for the answer to a user-provided question.
+3. An automated Wiki Q&A tool which asks the model to perform retrieve its own Wikipedia pages in order to answer the user-provided question.
+    ''')
 
-# Alternatively, you can try an example by clicking one of the buttons below:
-# ''')
+################################
+### Populate tab - basic Q&A ###
+################################
 
-# Generate containers in order
-# example_container = st.container()
-input_container = st.container()
-button_container = st.container()
-response_container = st.container()
+from lib.utils import basic_clear_boxes, basic_ex_click
 
-###########################
-### Populate containers ###
-###########################
+with basic_title_container:
+    ### Intro text ###
+    st.header('Basic extractive Q&A')
+    st.markdown('''
+The basic functionality of a RoBERTa model for extractive question-answering is to attempt to extract the answer to a user-provided question from a piece of user-provided context text.  The model is also trained to recognize when the context doesn't provide the answer.
 
-# Populate example button container
-# with example_container:
-#     ex_cols = st.columns(len(ex_q)+1)
-#     for i in range(len(ex_q)):
-#         with ex_cols[i]:
-#             st.button(
-#                 label = f'Try example {i+1}',
-#                 key = f'ex_button_{i+1}',
-#                 on_click = fill_in_example,
-#                 args=(i,),
-#             )
-#     with ex_cols[-1]:
-#         st.button(
-#             label = "Clear all fields",
-#             key = "clear_button",
-#             on_click = clear_boxes,
-#         )
+Please type or paste a context paragraph and question you'd like to ask about it.  The model will attempt to answer the question based on the context you provided, or report that it cannot find the answer in the context.  Your results will appear below the question field when the model is finished running.
 
-# Populate user input container
-with input_container:
-    with st.form(key='input_form',clear_on_submit=False):
+Alternatively, you can try an example by clicking one of the buttons below:
+    ''')
+    
+### Populate example button container ###
+with basic_example_container:
+    basic_ex_cols = st.columns(len(examples[0])+1)
+    for i in range(len(examples[0])):
+        with basic_ex_cols[i]:
+            st.button(
+                label = f'example {i+1}',
+                key = f'basic_ex_button_{i+1}',
+                on_click = basic_ex_click,
+                args = (examples,i,),
+            )
+    with basic_ex_cols[-1]:
+        st.button(
+            label = "Clear all fields",
+            key = "basic_clear_button",
+            on_click = basic_clear_boxes,
+        )
+### Populate user input container ###
+with basic_input_container:
+    with st.form(key='basic_input_form',clear_on_submit=False):
+        # Context input field
+        context = st.text_area(
+            label='Context',
+            value=st.session_state['basic']['context'],
+            key='basic_context_field',
+            label_visibility='collapsed',
+            placeholder='Enter your context paragraph here.',
+            height=300,
+        )
         # Question input field
         question = st.text_input(
             label='Question',
-            value=st.session_state['question'],
-            key='question_field',
-            label_visibility='hidden',
+            value=st.session_state['basic']['question'],
+            key='basic_question_field',
+            label_visibility='collapsed',
             placeholder='Enter your question here.',
         )
         # Form submit button
         query_submitted = st.form_submit_button("Submit")
-        if query_submitted:
+        if query_submitted and question!= '':
             # update question, context in session state
-            st.session_state['question'] = question
-            with st.spinner('Retrieving documentation...'):
-                query = generate_query(nlp,question)
-                retriever = ContextRetriever()
-                retriever.get_pageids(query)
-                retriever.get_pages()
-                retriever.get_paragraphs()
-                retriever.rank_paragraphs(query)
+            st.session_state['basic']['question'] = question
+            st.session_state['basic']['context'] = context
             with st.spinner('Generating response...'):
-                # Loop through best_paragraph contexts
-                # looking for answer in each
-                best_answer = ""
-                for context in retriever.best_paragraphs:
-                    input = {
-                        'context':context,
-                        'question':st.session_state['question'],
-                    }
-                    # Pass to QA pipeline
-                    response = qa_pipeline(**input)
-                    if response['answer']!='':
-                        best_answer = response['answer']
-                        best_context = context
-                        break
-            # Update response in session state
-            if best_answer == "":
-                st.session_state['response'] = "I cannot find the answer to your question."
-            else:
-                st.session_state['response'] = f"""
-                    My answer is: {best_answer}
-                    
-                    ...and here's where I found it:
-                    
-                    {best_context}
+                # Generate dictionary from inputs
+                query = {
+                    'context':st.session_state['basic']['context'],
+                    'question':st.session_state['basic']['question'],
+                }
+                # Pass to QA pipeline
+                response = qa_pipeline(**query)
+                answer = response['answer']
+                confidence = response['score']
+                # Reformat empty answer to message
+                if answer == '':
+                    answer = "I don't have an answer based on the context provided."
+                # Update response in session state
+                st.session_state['basic']['response'] = f"""
+                    Answer: {answer}\n
+                    Confidence: {confidence:.2%}
                 """
+### Populate response container ###
+with basic_response_container:
+    st.write(st.session_state['basic']['response'])
+            
+#################################
+### Populate tab - guided Q&A ###
+#################################
 
-# Button for clearing the form
-with button_container:
-    st.button(
+from lib.utils import (
+    semi_ex_query_click,
+    semi_ex_question_click,
+    semi_clear_query,
+    semi_clear_question,
+)
+    
+### Intro text ###
+with semi_title_container:
+    st.header('User-guided Wiki Q&A')
+    st.markdown('''
+This component allows you to perform a Wikipedia search for source material to feed as contexts to the RoBERTa question-answering model.
+    ''')
+    with st.expander("Click here to find out what's happening behind the scenes..."):
+        st.markdown('''
+    1. A Wikipedia search is performed using your query, resulting in a list of pages which then populate the drop-down menu.
+    2. The pages you select are retrieved and broken up into paragraphs.  Wikipedia queries and page collection use the [wikipedia library](https://pypi.org/project/wikipedia/), a wrapper for the [MediaWiki API](https://www.mediawiki.org/wiki/API).
+    3. The paragraphs are ranked in descending order of relevance to your question, using the [Okapi BM25 score](https://en.wikipedia.org/wiki/Okapi_BM25) as implemented in the [rank_bm25 library](https://github.com/dorianbrown/rank_bm25).
+    4. Among these ranked paragraphs, approximately the top 25% are fed as context to the RoBERTa model, from which it will attempt to extract the answer to your question.  The 'hit' having the highest confidence (prediction probability) from the model is reported as the answer.
+        ''')
+
+### Populate query container ###
+with semi_query_container:
+    st.markdown('First submit a search query, or choose one of the examples.')
+    semi_query_cols = st.columns(len(examples[0])+1)
+    # Buttons for query examples
+    for i in range(len(examples[0])):
+        with semi_query_cols[i]:
+            st.button(
+                label = f'query {i+1}',
+                key = f'semi_query_button_{i+1}',
+                on_click = semi_ex_query_click,
+                args=(examples,i,),
+            )
+    # Button for clearning query field
+    with semi_query_cols[-1]:
+        st.button(
+            label = "Clear query",
+            key = "semi_clear_query",
+            on_click = semi_clear_query,
+        )
+    # Search query input form
+    with st.form(key='semi_query_form',clear_on_submit=False):
+        query = st.text_input(
+            label='Search query',
+            value=st.session_state['semi']['query'],
+            key='semi_query_field',
+            label_visibility='collapsed',
+            placeholder='Enter your Wikipedia search query here.',
+        )
+        query_submitted = st.form_submit_button("Submit")
+
+        if query_submitted and query != '':
+            st.session_state['semi']['query'] = query
+            # Retrieve Wikipedia page list from
+            # search results and store in session state
+            with st.spinner('Retrieving Wiki pages...'):
+                retriever = ContextRetriever()
+                retriever.get_pageids(query)
+                st.session_state['semi']['page_options'] = retriever.pageids
+                st.session_state['semi']['selected_pages'] = []
+    
+### Populate page selection container ###
+with semi_page_container:
+    st.markdown('Next select any number of Wikipedia pages to provide to RoBERTa:')
+    # Page title selection form
+    with st.form(key='semi_page_form',clear_on_submit=False):
+        selected_pages = st.multiselect(
+                label = "Choose Wiki pages for Q&A model:",
+                options = st.session_state['semi']['page_options'],
+                default = st.session_state['semi']['selected_pages'],
+                label_visibility = 'collapsed',
+                key = "semi_page_selectbox",
+                format_func = lambda x:x[1],
+            )
+        pages_submitted = st.form_submit_button("Submit")
+        if pages_submitted:
+            st.session_state['semi']['selected_pages'] = selected_pages
+
+### Populate question input container ###
+with semi_input_container:
+    st.markdown('Finally submit a question for RoBERTa to answer based on the above pages or choose one of the examples.')
+    # Question example buttons
+    semi_ques_cols = st.columns(len(examples[0])+1)
+    for i in range(len(examples[0])):
+        with semi_ques_cols[i]:
+            st.button(
+                label = f'question {i+1}',
+                key = f'semi_ques_button_{i+1}',
+                on_click = semi_ex_question_click,
+                args=(i,),
+            )
+    # Question field clear button
+    with semi_ques_cols[-1]:
+        st.button(
+            label = "Clear question",
+            key = "semi_clear_question",
+            on_click = semi_clear_question,
+        )
+    # Question submission form
+    with st.form(key = "semi_question_form",clear_on_submit=False):
+        question = st.text_input(
+            label='Question',
+            value=st.session_state['semi']['question'],
+            key='semi_question_field',
+            label_visibility='collapsed',
+            placeholder='Enter your question here.',
+        )
+        question_submitted = st.form_submit_button("Submit")
+        if question_submitted and len(question)>0 and len(st.session_state['semi']['selected_pages'])>0:
+            st.session_state['semi']['response'] = ''
+            st.session_state['semi']['question'] = question
+            # Retrieve pages corresponding to user selections,
+            # extract paragraphs, and retrieve top 10 paragraphs,
+            # ranked by relevance to user question
+            with st.spinner("Retrieving documentation..."):
+                retriever = ContextRetriever()
+                pages = retriever.ids_to_pages(selected_pages)
+                paragraphs = retriever.pages_to_paragraphs(pages)
+                best_paragraphs = retriever.rank_paragraphs(
+                    paragraphs, question,
+                )     
+            with st.spinner("Generating response..."):
+                # Generate a response and update the session state
+                response = generate_answer(
+                    pipeline = qa_pipeline,
+                    paragraphs = best_paragraphs,
+                    question = st.session_state['semi']['question'],
+                )
+                st.session_state['semi']['response'] = response
+                
+### Populate response container ###
+with semi_response_container:
+    st.write(st.session_state['semi']['response'])
+
+####################################
+### Populate tab - automated Q&A ###
+####################################
+
+from lib.utils import auto_ex_click, auto_clear_boxes
+    
+### Intro text ###
+with auto_title_container:
+    st.header('Automated Wiki Q&A')
+    st.markdown('''
+This component attempts to automate the Wiki-assisted extractive question-answering task.  A Wikipedia search will be performed based on your question, and a list of relevant paragraphs will be passed to the RoBERTa model so it can attempt to find an answer.
+    ''')
+    with st.expander("Click here to find out what's happening behind the scenes..."):
+        st.markdown('''
+    When you submit a question, the following steps are performed:
+    1. Your question is condensed into a search query which just retains nouns, verbs, numerals, and adjectives, where part-of-speech tagging is done using the [en_core_web_sm](https://spacy.io/models/en#en_core_web_sm) pipeline in the [spaCy library](https://spacy.io/).
+    2. A Wikipedia search is performed using this query, resulting in several articles.  The articles from the top 3 search results are collected and split into paragraphs.  Wikipedia queries and article collection use the [wikipedia library](https://pypi.org/project/wikipedia/), a wrapper for the [MediaWiki API](https://www.mediawiki.org/wiki/API).
+    4. The paragraphs are ranked in descending order of relevance to the query, using the [Okapi BM25 score](https://en.wikipedia.org/wiki/Okapi_BM25) as implemented in the [rank_bm25 library](https://github.com/dorianbrown/rank_bm25).
+    5. The ten most relevant paragraphs are fed as context to the RoBERTa model, from which it will attempt to extract the answer to your question.  The 'hit' having the highest confidence (prediction probability) from the model is reported as the answer.
+        ''')
+
+    st.markdown('''
+Please provide a question you'd like the model to try to answer.  The model will report back its answer, as well as an excerpt of text from Wikipedia in which it found its answer.  Your result will appear below the question field when the model is finished running.
+
+Alternatively, you can try an example by clicking one of the buttons below:
+    ''')
+
+### Populate example container ###
+with auto_example_container:
+    auto_ex_cols = st.columns(len(examples[0])+1)
+    # Buttons for selecting example questions
+    for i in range(len(examples[0])):
+        with auto_ex_cols[i]:
+            st.button(
+                label = f'example {i+1}',
+                key = f'auto_ex_button_{i+1}',
+                on_click = auto_ex_click,
+                args=(examples,i,),
+            )
+    # Button for clearing question field and response
+    with auto_ex_cols[-1]:
+        st.button(
             label = "Clear all fields",
-            key = "clear_button",
-            on_click = clear_boxes,
+            key = "auto_clear_button",
+            on_click = auto_clear_boxes,
         )
 
-# Display response
-with response_container:
-    st.write(st.session_state['response'])
-            
+### Populate user input container ###
+with auto_input_container:
+    with st.form(key='auto_input_form',clear_on_submit=False):
+        # Question input field
+        question = st.text_input(
+            label='Question',
+            value=st.session_state['auto']['question'],
+            key='auto_question_field',
+            label_visibility='collapsed',                
+            placeholder='Enter your question here.',
+        )
+        # Form submit button
+        question_submitted = st.form_submit_button("Submit")
+        if question_submitted:
+            # update question, context in session state
+            st.session_state['auto']['question'] = question
+            query = generate_query(nlp,question)
+            # query == '' will throw error in document retrieval
+            if len(query)==0:
+                st.session_state['auto']['response'] = 'Please include some nouns, verbs, and/or adjectives in your question.'
+            elif len(question)>0:
+                with st.spinner('Retrieving documentation...'):
+                    # Retrieve ids from top 3 results
+                    retriever = ContextRetriever()
+                    retriever.get_pageids(query,topn=3)
+                    # Retrieve pages then paragraphs
+                    retriever.get_all_pages()
+                    retriever.get_all_paragraphs()
+                    # Get top 10 paragraphs, ranked by relevance to query
+                    best_paragraphs = retriever.rank_paragraphs(retriever.paragraphs, query)
+                with st.spinner('Generating response...'):
+                    # Generate a response and update the session state
+                    response = generate_answer(
+                        pipeline = qa_pipeline,
+                        paragraphs = best_paragraphs,
+                        question = st.session_state['auto']['question'],
+                    )
+                    st.session_state['auto']['response'] = response
+### Populate response container ###
+with auto_response_container:
+    st.write(st.session_state['auto']['response'])

examples.csv

@@ -0,0 +1,4 @@
+question,context,query
+What did Robert Oppenheimer remark after the Trinity test?:When was Robert Oppenheimer born?:What did Robert Oppenheimer do after World War II?,"Oppenheimer attended Harvard University, where he earned a bachelor's degree in chemistry in 1925. He studied physics at the University of Cambridge and University of Göttingen, where he received his PhD in 1927. He held academic positions at the University of California, Berkeley, and the California Institute of Technology, and made significant contributions to theoretical physics, including in quantum mechanics and nuclear physics. During World War II, he was recruited to work on the Manhattan Project, and in 1943 was appointed as director of the Los Alamos Laboratory in New Mexico, tasked with developing the weapons. Oppenheimer's leadership and scientific expertise were instrumental in the success of the project. He was among those who observed the Trinity test on July 16, 1945, in which the first atomic bomb was successfully detonated. He later remarked that the explosion brought to his mind words from the Hindu scripture Bhagavad Gita: ""Now I am become Death, the destroyer of worlds."" In August 1945, the atomic bombs were used on the Japanese cities of Hiroshima and Nagasaki, the only use of nuclear weapons in war.",Oppenheimer
+Why did Twinkies change to vanilla cream?:When were Twinkies invented?:What is the urban legend about Twinkies?,"Twinkies were invented on April 6, 1930, by Canadian-born baker James Alexander Dewar for the Continental Baking Company in Schiller Park, Illinois. Realizing that several machines used for making cream-filled strawberry shortcake sat idle when strawberries were out of season, Dewar conceived a snack cake filled with banana cream, which he dubbed the Twinkie. Ritchy Koph said he came up with the name when he saw a billboard in St. Louis for ""Twinkle Toe Shoes"".  During World War II, bananas were rationed, and the company was forced to switch to vanilla cream. This change proved popular, and banana-cream Twinkies were not widely re-introduced. The original flavor was occasionally found in limited time only promotions, but the company used vanilla cream for most Twinkies. In 1988, Fruit and Cream Twinkies were introduced with a strawberry filling swirled into the cream. The product was soon dropped. Vanilla's dominance over banana flavoring was challenged in 2005, following a month-long promotion of the movie King Kong. Hostess saw its Twinkie sales rise 20 percent during the promotion, and in 2007 restored the banana-cream Twinkie to its snack lineup although they are now made with 2% banana purée.",Twinkie
+When was Pinkfong founded?:What is Pinkfong's most famous song?:Who represents the company?,"""Baby Shark"" is a children's song associated with a dance involving hand movements that originated as a campfire song dating back to at least the 20th century. In 2016, ""Baby Shark"" became very popular when Pinkfong, a South Korean entertainment company, released a version of the song with a YouTube music video that went viral across social media, online video, and radio. In January 2022, it became the first YouTube video to reach 10 billion views. In November 2020, Pinkfong's version became the most-viewed YouTube video of all time, with over 12 billion views as of April 2023. ""Baby Shark"" originated as a campfire song or chant. The original song dates back to at least the 20th century, potentially created by camp counselors inspired by the movie Jaws. In the chant, each member of a family of sharks is introduced, with campers using their hands to imitate the sharks' jaws. Different versions of the song have the sharks hunting fish, eating a sailor, or killing people who then go to heaven. Various entities have copyrighted original videos and sound recordings of the song, and some have trademarked merchandise based on their versions. However, according to The New York Times, the underlying song and characters are believed to be in the public domain.",Pinkfong

lib/.ipynb_checkpoints/utils-checkpoint.py

@@ -1,42 +1,119 @@
-import requests, wikipedia, re, spacy
+import requests, wikipedia, re
 from rank_bm25 import BM25Okapi
-import torch
-from datasets import Dataset
-from torch.utils.data import DataLoader
-from transformers import (
-    AutoTokenizer,
-    AutoModelForQuestionAnswering,
-    pipeline,
-)
-
-class QueryProcessor:
-    """
-    Processes text into queries using a spaCy model
-    """
-    def __init__(self):
-        self.keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
-        self.nlp = spacy.load(
-            'en_core_web_sm',
-            disable = ['ner','parser','textcat']
-        )
-
-    def generate_query(self,text):
-        """
-        Process text into a search query,
-        only retaining nouns, proper nouns numerals, verbs, and adjectives
-        Parameters:
-        -----------
-        text : str
-            The input text to be processed
-        Returns:
-        --------
-        query : str
-            The condensed search query
-        """
-        tokens = self.nlp(text)
-        query = ' '.join(token.text for token in tokens \
-                         if token.pos_ in self.keep)
-        return query
+import streamlit as st
+import pandas as pd
+import spacy
+
+####################################
+## Streamlit app helper functions ##
+####################################
+
+def get_examples():
+    """
+    Function for loading example questions
+    and contexts from examples.csv
+    Parameters: None
+    -----------
+    Returns:
+    --------
+    ex_queries, ex_questions, ex_contexts : list(str), list(list(str)), list(str)
+        Example search query, question, and context strings
+        (each entry of ex_questions is a list of three question strings)
+    """
+    examples = pd.read_csv('examples.csv')
+    ex_questions = [q.split(':') for q in list(examples['question'])]
+    ex_contexts = list(examples['context'])
+    ex_queries = list(examples['query'])
+    return ex_queries, ex_questions, ex_contexts
+
+def basic_clear_boxes():
+    """
+    Clears the question, context, response
+    """
+    for field in ['question','context','response']:
+        st.session_state['basic'][field] = ''
+
+def basic_ex_click(examples, i):
+    """
+    Fills in the chosen example
+    """
+    st.session_state['basic']['question'] = examples[1][i][0]
+    st.session_state['basic']['context'] = examples[2][i]
+
+def semi_clear_query():
+    """
+    Clears the search query field
+    and page options list
+    """
+    st.session_state['semi']['query'] = ''
+    for field in ['selected_pages','page_options']:
+        st.session_state['semi'][field] = []
+
+def semi_clear_question():
+    """
+    Clears the question and response field
+    and selected pages list
+    """
+    for field in ['question','response']:
+        st.session_state['semi'][field] = ''
+
+def semi_ex_query_click(examples,i):
+    """
+    Fills in the query example and
+    populates question examples when query
+    example button is clicked
+    """
+    st.session_state['semi']['query'] = examples[0][i]
+    st.session_state['semi']['ex_questions'] = examples[1][i]
+
+def semi_ex_question_click(i):
+    """
+    Fills in the question example
+    """
+    st.session_state['semi']['question'] = st.session_state['semi']['ex_questions'][i]
+
+def auto_clear_boxes():
+    """
+    Clears the response and question fields
+    """
+    for field in ['question','response']:
+        st.session_state['auto'][field]=''
+
+def auto_ex_click(examples,i):
+    """
+    Fills in the chosen example question
+    """
+    st.session_state['auto']['question'] = examples[1][i][0]
+
+###########################
+## Query helper function ##
+###########################
+
+def generate_query(nlp,text):
+    """
+    Process text into a search query,
+    only retaining nouns, proper nouns,
+    numerals, verbs, and adjectives
+    Parameters:
+    -----------
+    nlp : spacy.Pipe
+        spaCy pipeline for processing search query
+    text : str
+        The input text to be processed
+    Returns:
+    --------
+    query : str
+        The condensed search query
+    """
+    tokens = nlp(text)
+    keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
+    query = ' '.join(token.text for token in tokens \
+                         if token.pos_ in keep)
+    return query
+        
+##############################
+## Document retriever class ##
+##############################
 
 class ContextRetriever:
     """
@@ -49,17 +126,21 @@ class ContextRetriever:
         self.pages = None
         self.paragraphs = None
         
-    def get_pageids(self,query):
+    def get_pageids(self,query,topn = None):
         """
         Retrieve page ids corresponding to a search query
         Parameters:
         -----------
         query : str
             A query to use for Wikipedia page search
+        topn : int or None
+            If topn is provided, will only return pageids
+            for topn search results
         Returns: None, but stores:
         --------
-        self.pageids : list(int)
-            A list of Wikipedia page ids corresponding to search results
+        self.pageids : list(tuple(int,str))
+            A list of Wikipedia (pageid,title) tuples resulting
+            from the search
         """
         params = {
             'action':'query',
@@ -68,39 +149,62 @@ class ContextRetriever:
             'format':'json',
         }
         results = requests.get(self.url, params=params).json()
-        pageids = [page['pageid'] for page in results['query']['search']]
+        pageids = [(page['pageid'],page['title']) for page in results['query']['search']]
+        pageids = pageids[:topn]
         self.pageids = pageids
-    
-    def get_pages(self):
+
+    def ids_to_pages(self,ids):
         """
         Use MediaWiki API to retrieve page content corresponding to
-        entries of self.pageids
-        Parameters: None
+        a list of pageids
+        Parameters:
         -----------
+        ids : list(tuple(int,str))
+            A list of Wikipedia (pageid,title) tuples
         Returns: None, but stores
         --------
-        self.pages : list(str)
-            Entries are content of pages corresponding to entries of self.pageid
+        pages : list(tuple(str,str))
+            The k-th enry is a tuple consisting of the title and page content
+            of the page corresponding to the k-th entry of ids
         """
-        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
-        self.pages = []
-        for pageid in self.pageids:
+        pages = []
+        for pageid in ids:
             try:
-                self.pages.append(wikipedia.page(pageid=pageid,auto_suggest=False).content)
-            except wikipedia.DisambiguationError as e:
+                page = wikipedia.page(pageid=pageid[0],auto_suggest=False)
+                pages.append((page.title, page.content))
+            except wikipedia.DisambiguationError:
                 continue
-
-    def get_paragraphs(self):
+        return pages
+    
+    def get_all_pages(self):
         """
-        Process self.pages into list of paragraphs from pages
+        Use MediaWiki API to retrieve page content corresponding to
+        the list of pageids in self.pageids
         Parameters: None
         -----------
         Returns: None, but stores
         --------
-        self.paragraphs : list(str)
-            List of paragraphs from all pages in self.pages, in order of self.pages
+        self.pages : list(tuple(str,str))
+            The k-th enry is a tuple consisting of the title and page content
+            of the page corresponding to the k-th entry of self.pageids
+        """
+        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
+        self.pages = self.ids_to_pages(self.pageids)
+
+    def pages_to_paragraphs(self,pages):
+        """
+        Process a list of pages into a list of paragraphs from those pages
+        Parameters:
+        -----------
+        pages : list(str)
+            A list of Wikipedia page content dumps, as strings
+        Returns:
+        --------
+        paragraphs : dict
+            keys are titles of pages from pages (as strings)
+            paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
         """
-        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
         # Content from WikiMedia has these headings. We only grab content appearing
         # before the first instance of any of these
         pattern = '|'.join([
@@ -117,38 +221,144 @@ class ContextRetriever:
             '=== Citations ===',
         ])
         pattern = re.compile(pattern)
-        paragraphs = []
-        for page in self.pages:
+        paragraphs = {}
+        for page in pages:
             # Truncate page to the first index of the start of a matching heading,
             # or the end of the page if no matches exist
-            idx = min([match.start() for match in pattern.finditer(page)]+[len(page)])
-            page = page[:idx]
+            title, content = page
+            idx = min([match.start() for match in pattern.finditer(content)]+[len(content)])
+            content = content[:idx]
             # Split into paragraphs, omitting lines with headings (start with '='),
             # empty lines, or lines like '\t\t' or '\t\t\t' which sometimes appear
-            paragraphs += [
-                p for p in page.split('\n') if p \
+            paragraphs[title] = [
+                p for p in content.split('\n') if p \
                 and not p.startswith('=') \
-                and not p.startswith('\t\t')
+                and not p.startswith('\t\t') \
+                and not p.startswith('  ')
             ]
-        self.paragraphs = paragraphs
+        return paragraphs
+    
+    def get_all_paragraphs(self):
+        """
+        Process self.pages into list of paragraphs from pages
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.paragraphs : dict
+            keys are titles of pages from self.pages (as strings)
+            self.paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
+        """
+        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
+        # Content from WikiMedia has these headings. We only grab content appearing
+        # before the first instance of any of these
+        self.paragraphs = self.pages_to_paragraphs(self.pages)
 
-    def rank_paragraphs(self,query,topn=10):
+    def rank_paragraphs(self,paragraphs,query,topn=10):
         """
-        Ranks the elements of self.paragraphs in descending order
-        by relevance to query using BM25F, and returns top topn results
+        Ranks the elements of paragraphs in descending order
+        by relevance to query using BM25 Okapi, and returns top
+        topn results
         Parameters:
         -----------
+        paragraphs : dict
+            keys are titles of pages (as strings)
+            paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
         query : str
             The query to use in ranking paragraphs by relevance
-        topn : int
+        topn : int or None
             The number of most relevant paragraphs to return
-        Returns: None, but stores
+            If None, will return roughly the top 1/4 of the
+            paragraphs
+        Returns:
         --------
-        self.best_paragraphs : list(str)
-            The topn most relevant paragraphs to the query
+        best_paragraphs : list(list(str,str))
+            The k-th entry is a list [title,paragraph] for the k-th
+            most relevant paragraph, where title is the title of the
+            Wikipedia article from which that paragraph was sourced
         """
-        tokenized_paragraphs = [p.split(" ") for p in self.paragraphs]
-        bm25 = BM25Okapi(tokenized_paragraphs)
+        corpus, titles, page_nums = [],[],[]
+        # Compile paragraphs into corpus
+        for i,page in enumerate(paragraphs):
+            titles.append(page)
+            paras = paragraphs[page]
+            corpus += paras
+            page_nums += len(paras)*[i]
+
+        # Tokenize corpus and query and initialize bm25 object
+        tokenized_corpus = [p.split(" ") for p in corpus]
+        bm25 = BM25Okapi(tokenized_corpus)
         tokenized_query = query.split(" ")
-        self.best_paragraphs = bm25.get_top_n(tokenized_query,self.paragraphs,n=topn)
 
+        # Compute scores and compile tuples (paragraph number, score, page number)
+        # before sorting tuples by score
+        bm_scores = bm25.get_scores(tokenized_query)
+        paragraph_data = [[i,score,page_nums[i]] for i,score in enumerate(bm_scores)]
+        paragraph_data.sort(reverse=True,key=lambda p:p[1])
+        
+        # Grab topn best [title,paragraph] pairs sorted by bm25 score
+        topn = len(paragraph_data)//4+1 if topn is None else min(topn,len(paragraph_data))
+
+        best_paragraphs = [[titles[p[2]],corpus[p[0]]] for p in paragraph_data[:topn]]
+        return best_paragraphs
+
+def generate_answer(pipeline,paragraphs, question):
+    """
+    Generate an answer using a question-answer pipeline
+    Parameters:
+    -----------
+    pipeline : transformers.QuestionAnsweringPipeline
+        The question answering pipeline object
+    paragraphs : list(list(str,str))
+        The k-th entry is a list [title,paragraph] consisting
+        of a context paragraph and the title of the page from which the
+        paragraph was sourced
+    question : str
+        A question that is to be answered based on context given
+        in the entries of paragraphs
+
+    Returns:
+    --------
+    response : str
+        A response indicating the answer that was discovered,
+        or indicating that no answer could be found.
+    """
+    # For each paragraph, format input to QA pipeline...
+    for paragraph in paragraphs:
+        input = {
+            'context':paragraph[1],
+            'question':question,
+        }
+        # ...and pass to QA pipeline
+        output = pipeline(**input)
+        # Append answers and scores.  Report score of
+        # zero for paragraphs without answer, so they are
+        # deprioritized when the max is taken below
+        if output['answer']!='':
+            paragraph += [output['answer'],output['score']]               
+        else:
+            paragraph += ['',0]
+    # Get paragraph with max confidence score and collect data
+    best_paragraph = max(paragraphs,key = lambda x:x[3])
+    best_answer = best_paragraph[2]
+    best_context_page = best_paragraph[0]
+    best_context = best_paragraph[1]
+                
+    # Update response in session state
+    if best_answer == "":
+        response = "I cannot find the answer to your question."
+    else:
+        response = f"""
+            My answer is: {best_answer}
+                    
+            ...and here's where I found it:
+                    
+            Page title: {best_context_page}
+
+            Paragraph containing answer:
+
+            {best_context}
+            """
+    return response

lib/utils.py

@@ -1,13 +1,119 @@
 import requests, wikipedia, re
 from rank_bm25 import BM25Okapi
-import torch
-from datasets import Dataset
-from torch.utils.data import DataLoader
-from transformers import (
-    AutoTokenizer,
-    AutoModelForQuestionAnswering,
-    pipeline,
-)
+import streamlit as st
+import pandas as pd
+import spacy
+
+####################################
+## Streamlit app helper functions ##
+####################################
+
+def get_examples():
+    """
+    Function for loading example questions
+    and contexts from examples.csv
+    Parameters: None
+    -----------
+    Returns:
+    --------
+    ex_queries, ex_questions, ex_contexts : list(str), list(list(str)), list(str)
+        Example search query, question, and context strings
+        (each entry of ex_questions is a list of three question strings)
+    """
+    examples = pd.read_csv('examples.csv')
+    ex_questions = [q.split(':') for q in list(examples['question'])]
+    ex_contexts = list(examples['context'])
+    ex_queries = list(examples['query'])
+    return ex_queries, ex_questions, ex_contexts
+
+def basic_clear_boxes():
+    """
+    Clears the question, context, response
+    """
+    for field in ['question','context','response']:
+        st.session_state['basic'][field] = ''
+
+def basic_ex_click(examples, i):
+    """
+    Fills in the chosen example
+    """
+    st.session_state['basic']['question'] = examples[1][i][0]
+    st.session_state['basic']['context'] = examples[2][i]
+
+def semi_clear_query():
+    """
+    Clears the search query field
+    and page options list
+    """
+    st.session_state['semi']['query'] = ''
+    for field in ['selected_pages','page_options']:
+        st.session_state['semi'][field] = []
+
+def semi_clear_question():
+    """
+    Clears the question and response field
+    and selected pages list
+    """
+    for field in ['question','response']:
+        st.session_state['semi'][field] = ''
+
+def semi_ex_query_click(examples,i):
+    """
+    Fills in the query example and
+    populates question examples when query
+    example button is clicked
+    """
+    st.session_state['semi']['query'] = examples[0][i]
+    st.session_state['semi']['ex_questions'] = examples[1][i]
+
+def semi_ex_question_click(i):
+    """
+    Fills in the question example
+    """
+    st.session_state['semi']['question'] = st.session_state['semi']['ex_questions'][i]
+
+def auto_clear_boxes():
+    """
+    Clears the response and question fields
+    """
+    for field in ['question','response']:
+        st.session_state['auto'][field]=''
+
+def auto_ex_click(examples,i):
+    """
+    Fills in the chosen example question
+    """
+    st.session_state['auto']['question'] = examples[1][i][0]
+
+###########################
+## Query helper function ##
+###########################
+
+def generate_query(nlp,text):
+    """
+    Process text into a search query,
+    only retaining nouns, proper nouns,
+    numerals, verbs, and adjectives
+    Parameters:
+    -----------
+    nlp : spacy.Pipe
+        spaCy pipeline for processing search query
+    text : str
+        The input text to be processed
+    Returns:
+    --------
+    query : str
+        The condensed search query
+    """
+    tokens = nlp(text)
+    keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
+    query = ' '.join(token.text for token in tokens \
+                         if token.pos_ in keep)
+    return query
+        
+##############################
+## Document retriever class ##
+##############################
 
 class ContextRetriever:
     """
@@ -20,17 +126,21 @@ class ContextRetriever:
         self.pages = None
         self.paragraphs = None
         
-    def get_pageids(self,query):
+    def get_pageids(self,query,topn = None):
         """
         Retrieve page ids corresponding to a search query
         Parameters:
         -----------
         query : str
             A query to use for Wikipedia page search
+        topn : int or None
+            If topn is provided, will only return pageids
+            for topn search results
         Returns: None, but stores:
         --------
-        self.pageids : list(int)
-            A list of Wikipedia page ids corresponding to search results
+        self.pageids : list(tuple(int,str))
+            A list of Wikipedia (pageid,title) tuples resulting
+            from the search
         """
         params = {
             'action':'query',
@@ -39,39 +149,62 @@ class ContextRetriever:
             'format':'json',
         }
         results = requests.get(self.url, params=params).json()
-        pageids = [page['pageid'] for page in results['query']['search']]
+        pageids = [(page['pageid'],page['title']) for page in results['query']['search']]
+        pageids = pageids[:topn]
         self.pageids = pageids
-    
-    def get_pages(self):
+
+    def ids_to_pages(self,ids):
         """
         Use MediaWiki API to retrieve page content corresponding to
-        entries of self.pageids
-        Parameters: None
+        a list of pageids
+        Parameters:
         -----------
+        ids : list(tuple(int,str))
+            A list of Wikipedia (pageid,title) tuples
         Returns: None, but stores
         --------
-        self.pages : list(str)
-            Entries are content of pages corresponding to entries of self.pageid
+        pages : list(tuple(str,str))
+            The k-th enry is a tuple consisting of the title and page content
+            of the page corresponding to the k-th entry of ids
         """
-        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
-        self.pages = []
-        for pageid in self.pageids:
+        pages = []
+        for pageid in ids:
             try:
-                self.pages.append(wikipedia.page(pageid=pageid,auto_suggest=False).content)
-            except wikipedia.DisambiguationError as e:
+                page = wikipedia.page(pageid=pageid[0],auto_suggest=False)
+                pages.append((page.title, page.content))
+            except wikipedia.DisambiguationError:
                 continue
-
-    def get_paragraphs(self):
+        return pages
+    
+    def get_all_pages(self):
         """
-        Process self.pages into list of paragraphs from pages
+        Use MediaWiki API to retrieve page content corresponding to
+        the list of pageids in self.pageids
         Parameters: None
         -----------
         Returns: None, but stores
         --------
-        self.paragraphs : list(str)
-            List of paragraphs from all pages in self.pages, in order of self.pages
+        self.pages : list(tuple(str,str))
+            The k-th enry is a tuple consisting of the title and page content
+            of the page corresponding to the k-th entry of self.pageids
+        """
+        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
+        self.pages = self.ids_to_pages(self.pageids)
+
+    def pages_to_paragraphs(self,pages):
+        """
+        Process a list of pages into a list of paragraphs from those pages
+        Parameters:
+        -----------
+        pages : list(str)
+            A list of Wikipedia page content dumps, as strings
+        Returns:
+        --------
+        paragraphs : dict
+            keys are titles of pages from pages (as strings)
+            paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
         """
-        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
         # Content from WikiMedia has these headings. We only grab content appearing
         # before the first instance of any of these
         pattern = '|'.join([
@@ -88,38 +221,144 @@ class ContextRetriever:
             '=== Citations ===',
         ])
         pattern = re.compile(pattern)
-        paragraphs = []
-        for page in self.pages:
+        paragraphs = {}
+        for page in pages:
             # Truncate page to the first index of the start of a matching heading,
             # or the end of the page if no matches exist
-            idx = min([match.start() for match in pattern.finditer(page)]+[len(page)])
-            page = page[:idx]
+            title, content = page
+            idx = min([match.start() for match in pattern.finditer(content)]+[len(content)])
+            content = content[:idx]
             # Split into paragraphs, omitting lines with headings (start with '='),
             # empty lines, or lines like '\t\t' or '\t\t\t' which sometimes appear
-            paragraphs += [
-                p for p in page.split('\n') if p \
+            paragraphs[title] = [
+                p for p in content.split('\n') if p \
                 and not p.startswith('=') \
-                and not p.startswith('\t\t')
+                and not p.startswith('\t\t') \
+                and not p.startswith('  ')
             ]
-        self.paragraphs = paragraphs
+        return paragraphs
+    
+    def get_all_paragraphs(self):
+        """
+        Process self.pages into list of paragraphs from pages
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.paragraphs : dict
+            keys are titles of pages from self.pages (as strings)
+            self.paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
+        """
+        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
+        # Content from WikiMedia has these headings. We only grab content appearing
+        # before the first instance of any of these
+        self.paragraphs = self.pages_to_paragraphs(self.pages)
 
-    def rank_paragraphs(self,query,topn=10):
+    def rank_paragraphs(self,paragraphs,query,topn=10):
         """
-        Ranks the elements of self.paragraphs in descending order
-        by relevance to query using BM25F, and returns top topn results
+        Ranks the elements of paragraphs in descending order
+        by relevance to query using BM25 Okapi, and returns top
+        topn results
         Parameters:
         -----------
+        paragraphs : dict
+            keys are titles of pages (as strings)
+            paragraphs[page] is a list of paragraphs (as strings)
+            extracted from page
         query : str
             The query to use in ranking paragraphs by relevance
-        topn : int
+        topn : int or None
             The number of most relevant paragraphs to return
-        Returns: None, but stores
+            If None, will return roughly the top 1/4 of the
+            paragraphs
+        Returns:
         --------
-        self.best_paragraphs : list(str)
-            The topn most relevant paragraphs to the query
+        best_paragraphs : list(list(str,str))
+            The k-th entry is a list [title,paragraph] for the k-th
+            most relevant paragraph, where title is the title of the
+            Wikipedia article from which that paragraph was sourced
         """
-        tokenized_paragraphs = [p.split(" ") for p in self.paragraphs]
-        bm25 = BM25Okapi(tokenized_paragraphs)
+        corpus, titles, page_nums = [],[],[]
+        # Compile paragraphs into corpus
+        for i,page in enumerate(paragraphs):
+            titles.append(page)
+            paras = paragraphs[page]
+            corpus += paras
+            page_nums += len(paras)*[i]
+
+        # Tokenize corpus and query and initialize bm25 object
+        tokenized_corpus = [p.split(" ") for p in corpus]
+        bm25 = BM25Okapi(tokenized_corpus)
         tokenized_query = query.split(" ")
-        self.best_paragraphs = bm25.get_top_n(tokenized_query,self.paragraphs,n=topn)
 
+        # Compute scores and compile tuples (paragraph number, score, page number)
+        # before sorting tuples by score
+        bm_scores = bm25.get_scores(tokenized_query)
+        paragraph_data = [[i,score,page_nums[i]] for i,score in enumerate(bm_scores)]
+        paragraph_data.sort(reverse=True,key=lambda p:p[1])
+        
+        # Grab topn best [title,paragraph] pairs sorted by bm25 score
+        topn = len(paragraph_data)//4+1 if topn is None else min(topn,len(paragraph_data))
+
+        best_paragraphs = [[titles[p[2]],corpus[p[0]]] for p in paragraph_data[:topn]]
+        return best_paragraphs
+
+def generate_answer(pipeline,paragraphs, question):
+    """
+    Generate an answer using a question-answer pipeline
+    Parameters:
+    -----------
+    pipeline : transformers.QuestionAnsweringPipeline
+        The question answering pipeline object
+    paragraphs : list(list(str,str))
+        The k-th entry is a list [title,paragraph] consisting
+        of a context paragraph and the title of the page from which the
+        paragraph was sourced
+    question : str
+        A question that is to be answered based on context given
+        in the entries of paragraphs
+
+    Returns:
+    --------
+    response : str
+        A response indicating the answer that was discovered,
+        or indicating that no answer could be found.
+    """
+    # For each paragraph, format input to QA pipeline...
+    for paragraph in paragraphs:
+        input = {
+            'context':paragraph[1],
+            'question':question,
+        }
+        # ...and pass to QA pipeline
+        output = pipeline(**input)
+        # Append answers and scores.  Report score of
+        # zero for paragraphs without answer, so they are
+        # deprioritized when the max is taken below
+        if output['answer']!='':
+            paragraph += [output['answer'],output['score']]               
+        else:
+            paragraph += ['',0]
+    # Get paragraph with max confidence score and collect data
+    best_paragraph = max(paragraphs,key = lambda x:x[3])
+    best_answer = best_paragraph[2]
+    best_context_page = best_paragraph[0]
+    best_context = best_paragraph[1]
+                
+    # Update response in session state
+    if best_answer == "":
+        response = "I cannot find the answer to your question."
+    else:
+        response = f"""
+            My answer is: {best_answer}
+                    
+            ...and here's where I found it:
+                    
+            Page title: {best_context_page}
+
+            Paragraph containing answer:
+
+            {best_context}
+            """
+    return response