Spaces:
Sleeping
Sleeping
import streamlit as st | |
from transformers import T5ForConditionalGeneration, T5Tokenizer | |
import spacy | |
import nltk | |
from sklearn.feature_extraction.text import TfidfVectorizer | |
from rake_nltk import Rake | |
import pandas as pd | |
from fpdf import FPDF | |
import wikipediaapi | |
from functools import lru_cache | |
nltk.download('punkt') | |
nltk.download('stopwords') | |
nltk.download('brown') | |
from nltk.tokenize import sent_tokenize | |
nltk.download('wordnet') | |
from nltk.corpus import wordnet | |
import random | |
from sense2vec import Sense2Vec | |
import sense2vec | |
from wordcloud import WordCloud | |
import matplotlib.pyplot as plt | |
print("***************************************************************") | |
st.set_page_config( | |
page_title="Question Generator", | |
initial_sidebar_state="collapsed", | |
) | |
# Load spaCy model | |
nlp = spacy.load("en_core_web_md") | |
# s2v = Sense2Vec.from_disk(self=Sense2Vec,path='s2v_old') | |
s2v = sense2vec.Sense2Vec().from_disk('s2v_old') | |
# Initialize Wikipedia API with a user agent | |
user_agent = 'QGen/1.0 (channingfisher7@gmail.com)' | |
wiki_wiki = wikipediaapi.Wikipedia(user_agent= user_agent,language='en') | |
def load_model(): | |
model_name = "DevBM/t5-large-squad" | |
model = T5ForConditionalGeneration.from_pretrained(model_name) | |
tokenizer = T5Tokenizer.from_pretrained(model_name) | |
return model, tokenizer | |
# Function to extract keywords using combined techniques | |
def extract_keywords(text, extract_all): | |
doc = nlp(text) | |
spacy_keywords = set([ent.text for ent in doc.ents]) | |
spacy_entities = spacy_keywords | |
print(f"\n\nSpacy Entities: {spacy_entities} \n\n") | |
# Use Only Spacy Entities | |
if extract_all is False: | |
return list(spacy_entities) | |
# Use RAKE | |
rake = Rake() | |
rake.extract_keywords_from_text(text) | |
rake_keywords = set(rake.get_ranked_phrases()) | |
print(f"\n\nRake Keywords: {rake_keywords} \n\n") | |
# Use spaCy for NER and POS tagging | |
spacy_keywords.update([token.text for token in doc if token.pos_ in ["NOUN", "PROPN", "VERB", "ADJ"]]) | |
print(f"\n\nSpacy Keywords: {spacy_keywords} \n\n") | |
# Use TF-IDF | |
vectorizer = TfidfVectorizer(stop_words='english') | |
X = vectorizer.fit_transform([text]) | |
tfidf_keywords = set(vectorizer.get_feature_names_out()) | |
print(f"\n\nTFIDF Entities: {tfidf_keywords} \n\n") | |
# Combine all keywords | |
combined_keywords = rake_keywords.union(spacy_keywords).union(tfidf_keywords) | |
return list(combined_keywords) | |
def get_similar_words_sense2vec(word, n=3): | |
# Try to find the word with its most likely part-of-speech | |
word_with_pos = word + "|NOUN" | |
if word_with_pos in s2v: | |
similar_words = s2v.most_similar(word_with_pos, n=n) | |
return [word.split("|")[0] for word, _ in similar_words] | |
# If not found, try without POS | |
if word in s2v: | |
similar_words = s2v.most_similar(word, n=n) | |
return [word.split("|")[0] for word, _ in similar_words] | |
return [] | |
def get_synonyms(word, n=3): | |
synonyms = [] | |
for syn in wordnet.synsets(word): | |
for lemma in syn.lemmas(): | |
if lemma.name() != word and lemma.name() not in synonyms: | |
synonyms.append(lemma.name()) | |
if len(synonyms) == n: | |
return synonyms | |
return synonyms | |
def generate_options(answer, context, n=3): | |
options = [answer] | |
# Try to get similar words based on sense2vec | |
similar_words = get_similar_words_sense2vec(answer, n) | |
options.extend(similar_words) | |
# If we don't have enough options, try synonyms | |
if len(options) < n + 1: | |
synonyms = get_synonyms(answer, n - len(options) + 1) | |
options.extend(synonyms) | |
# If we still don't have enough options, extract other entities from the context | |
if len(options) < n + 1: | |
doc = nlp(context) | |
entities = [ent.text for ent in doc.ents if ent.text.lower() != answer.lower()] | |
options.extend(entities[:n - len(options) + 1]) | |
# If we still need more options, add some random words from the context | |
if len(options) < n + 1: | |
context_words = [token.text for token in nlp(context) if token.is_alpha and token.text.lower() != answer.lower()] | |
options.extend(random.sample(context_words, min(n - len(options) + 1, len(context_words)))) | |
# Ensure we have the correct number of unique options | |
options = list(dict.fromkeys(options))[:n+1] | |
# Shuffle the options | |
random.shuffle(options) | |
return options | |
# Function to map keywords to sentences with customizable context window size | |
def map_keywords_to_sentences(text, keywords, context_window_size): | |
sentences = sent_tokenize(text) | |
keyword_sentence_mapping = {} | |
for keyword in keywords: | |
for i, sentence in enumerate(sentences): | |
if keyword in sentence: | |
# Combine current sentence with surrounding sentences for context | |
start = max(0, i - context_window_size) | |
end = min(len(sentences), i + context_window_size + 1) | |
context = ' '.join(sentences[start:end]) | |
if keyword not in keyword_sentence_mapping: | |
keyword_sentence_mapping[keyword] = context | |
else: | |
keyword_sentence_mapping[keyword] += ' ' + context | |
return keyword_sentence_mapping | |
# Function to perform entity linking using Wikipedia API | |
def entity_linking(keyword): | |
page = wiki_wiki.page(keyword) | |
if page.exists(): | |
return page.fullurl | |
return None | |
# Function to generate questions using beam search | |
def generate_question(context, answer, num_beams): | |
input_text = f"<context> {context} <answer> {answer}" | |
input_ids = tokenizer.encode(input_text, return_tensors='pt') | |
outputs = model.generate(input_ids, num_beams=num_beams, early_stopping=True) | |
question = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
return question | |
# Function to export questions to CSV | |
def export_to_csv(data): | |
df = pd.DataFrame(data, columns=["Context", "Answer", "Question", "Options"]) | |
csv = df.to_csv(index=False,encoding='utf-8') | |
return csv | |
# Function to export questions to PDF | |
def export_to_pdf(data): | |
pdf = FPDF() | |
pdf.add_page() | |
pdf.set_font("Arial", size=12) | |
for context, answer, question, options in data: | |
pdf.multi_cell(0, 10, f"Context: {context}") | |
pdf.multi_cell(0, 10, f"Answer: {answer}") | |
pdf.multi_cell(0, 10, f"Question: {question}") | |
pdf.ln(10) | |
# pdf.output("questions.pdf") | |
return pdf.output(name='questions.pdf',dest='S').encode('latin1') | |
def display_word_cloud(generated_questions): | |
word_frequency = {} | |
for question in generated_questions: | |
words = question.split() | |
for word in words: | |
word_frequency[word] = word_frequency.get(word, 0) + 1 | |
wordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(word_frequency) | |
plt.figure(figsize=(10, 5)) | |
plt.imshow(wordcloud, interpolation='bilinear') | |
plt.axis('off') | |
st.pyplot() | |
if 'data' not in st.session_state: | |
st.session_state.data = None | |
# Streamlit interface | |
st.title(":blue[Question Generator from Text]") | |
text = st.text_area("Enter text here:", value="Joe Biden, the current US president is on a weak wicket going in for his reelection later this November against former President Donald Trump.") | |
with st.sidebar: | |
st.subheader("Customization Options") | |
# Customization options | |
num_beams = st.slider("Select number of beams for question generation", min_value=1, max_value=10, value=5) | |
context_window_size = st.slider("Select context window size (number of sentences before and after)", min_value=1, max_value=5, value=1) | |
num_questions = st.slider("Select number of questions to generate", min_value=1, max_value=1000, value=5) | |
with st.expander("Choose the Additional Elements to show"): | |
show_context = st.checkbox("Context",True) | |
show_answer = st.checkbox("Answer",True) | |
show_options = st.checkbox("Options",False) | |
show_entity_link = st.checkbox("Enitity Link For Wikipedia",True) | |
extract_all_keywords = st.toggle("Extract max Keywords",value=False) | |
if st.button("Generate Questions"): | |
if text: | |
model, tokenizer = load_model() | |
keywords = extract_keywords(text,extract_all_keywords) | |
print(f"\n\nFinal Keywords in Main Function: {keywords}\n\n") | |
keyword_sentence_mapping = map_keywords_to_sentences(text, keywords, context_window_size) | |
st.subheader("Generated Questions:",divider='blue') | |
data = [] | |
for i, (keyword, context) in enumerate(keyword_sentence_mapping.items()): | |
if i >= num_questions: | |
break | |
linked_entity = entity_linking(keyword) | |
question = generate_question(context, keyword, num_beams=num_beams) | |
options = generate_options(keyword, context) | |
st.subheader(body=f":orange[Q{i+1}:] {question}") | |
if show_context is True: | |
st.write(f"**Context:** {context}") | |
if show_answer is True: | |
st.write(f"**Answer:** {keyword}") | |
if show_options is True: | |
st.write(f"**Options:**") | |
for j, option in enumerate(options): | |
st.write(f"{chr(65+j)}. {option}") | |
if show_entity_link is True: | |
if linked_entity: | |
st.write(f"**Entity Link:** {linked_entity}") | |
st.write("---") | |
data.append((context, keyword, question, options)) | |
# Add the data to session state | |
st.session_state.data = data | |
# display_word_cloud() | |
print(data) | |
# Export buttons | |
if st.session_state.data is not None: | |
with st.sidebar: | |
st.subheader('Download Content') | |
csv_data = export_to_csv(data) | |
st.download_button(label="CSV Format", data=csv_data, file_name='questions.csv', mime='text/csv') | |
pdf_data = export_to_pdf(data) | |
st.download_button(label="PDF Format", data=pdf_data, file_name='questions.pdf', mime='application/pdf') | |
if st.session_state.data is not None: | |
st.markdown("You can download the data from the sidebar.") | |
else: | |
st.write("Please enter some text to generate questions.") | |
print("********************************************************************************") |