|
from re import split
|
|
import streamlit as st
|
|
import more_itertools
|
|
import spacy
|
|
from sentence_transformers import SentenceTransformer, util
|
|
|
|
st.set_page_config(
|
|
page_title="Semantic_Search/Summarization")
|
|
|
|
|
|
choclate_example = """You’d be hard-pressed to find anyone who doesn’t like chocolate. While it's mostly known for its taste (and the associated cravings), it’s also a good source of nutrients when in its pure form and eaten in moderation.
|
|
|
|
This well-loved food, once called the "drink of the gods" by the Maya people, has a rich history as well. Chocolate comes from the seeds of the cacao pod, which grows on the cacao tree. Theobroma cacao is native to the tropical rainforests of Central America, where it has grown for thousands of years.
|
|
|
|
"""
|
|
|
|
|
|
|
|
nlp = spacy.load('en_core_web_sm')
|
|
|
|
|
|
|
|
wit_text = """
|
|
|
|
. .
|
|
. .
|
|
.
|
|
. .. . .*/, . . .
|
|
.*&#&@@@&@@@@&%&&&@&(. .
|
|
*#&&@&&@@@@@@@@@%@@@@@@&,... . . . . .
|
|
#@@@@@@@@@@@@@@@@@@@@@@@@##... .. .. .
|
|
,&@&&@@&&&@@@@@%*........,&@@@&(,,...... .
|
|
.&&&@@&/,...... . ......(@@@@@&#...... . . .
|
|
.&@@&,... ... .. . ......#@@@@@&....... . .
|
|
.&@@&,... .. .............,,*@@@@@(....... . .
|
|
.@@@@/*..****,,*,*(&@@@@%(,...&@@@/....... . ....
|
|
,@@&,,/&(&@@@@(..(%(,&%,((,...&@&......... . . .. .
|
|
(@(..(%**,,(/....,,.............%/........ .. . .
|
|
../...........................(#.......... . . . .
|
|
.(.(...,....,/@@&&&#......,,,............ .. .....
|
|
..,.,,,,.....,,,*,........,..,.......... .. .. . .
|
|
../,**,,,.../@@@@@%,.,.,,,,............ . . . .
|
|
. .,**,,*#,**#%%/,,,,,,,,,,. ...... . .. .
|
|
. ..**********,......,,,**,. . ...... . .
|
|
. ....,/(/***,,,**,,,,*/((**,...... .. .
|
|
.. ....,/#&@@@@@@@@@@&%%#/**,........ .
|
|
....,/*,.,,**/#%&@@@@%#(%#/*,%..(*#(*..
|
|
.,(/*,(/#/(..,,,*///(////(((***,..*###%/%/(/,,. .
|
|
....,,*(((**/**/#(...,.,,*/(##/,,***/.@..%&#%%///(##/,*,,,. .
|
|
.....,,,*%(,&%(/////#*....,.,,***,,,*/(.@..&&%&#%/***#%(,(((*,*,,. ..
|
|
..*...,,,*,#%,*%(/*(/*#(##..&,..,,.,**,,*/,.#.&&&&&###((//%%/*****,,,,*.,,
|
|
...,,(*,,.,.*&&&%((**#((/(/&,.&..../(////,,..@@@@&&&&%(//,*(##(**/*,*,,/****,
|
|
,,/**/#(//*(%%/%@@@%(*/(/%##/,@.....,,*,...(@@@&%%(##(*/@@&((/(#/*/((%/*//***
|
|
,,/&/(#(###*/#&**#/%#///*//#//&@(... ....&@&&&###(%#####/%((*(#%%(#&&#%,((%&/
|
|
.,(,,*&%*/%&#(&((//#/,*/*/(/*//#&&/,.&.. ./&(((%%%#((%/(#(##,%*#/((%@%(##%&&*/(
|
|
.*,,**@#**%%&&#%%#(#/*/***/*/**(//&... , .%####%%&%(#/(%/(/(%&%#(#%@%%(&@(,*%%(
|
|
***(@%,,&@&##&%(@(/(//,,///(//*/**%.. ....#((###%&##(/%(&,(&@&#%###&&%@/@&(&%#&#
|
|
**/#&(((&&#@@@#%%&*/(**///(/,/,**/*.. ,*##/(((&%(%%#((%%&@@@#####&&@@&(%&@&#/(%%
|
|
(%#(*,%@@@(@@@&%%@&//(//(////**,/(#(,*(&%(%#%%&(%%#&(%(#@@@@&&%%%&@@@&/(&&&/*%&%
|
|
,*#&@@@@@&/(%@@%&&&%#(#//#/#/****##%*/*/*%&&%&&%%#%%(%@@@&@#%%&&@@@&&@@*#%&#(***
|
|
#(/***//%(((((&@&&&@&(/#(**(*((//**#/,/%&%&%&&%&##%%&&&@@@&#%&&&@@%(#*(*((%(%%&%
|
|
*,#%&&&@%&&&%(&@@&&&%&/%//(***/(///%(@&#&&&%&&&%&%@&&&##@@@&&&@@@&&((((###%(#/(*
|
|
/#**..,,*,/*#&@&@@&%&@%#(((#**(/(//%@@#&@#&&%#%%@@&&@@@@@@&@@@@@&#(###%##%%(%&#/
|
|
#%&%(,*,##&&&&&@@@&&@&&#*/*/(/*((//#@&(@&&%&&%&@&&&&@&@@@@@@&&@&&%#%(%(%#&%%#/(%
|
|
***,/((&%(/(**//&@@&@@@@(*/(,((#/((&@%%&#%&&&@&&#&&@@@@@@&@&@@@&((#&%#%%////(###
|
|
*,/*/#&&@@&@&*,%@@@@@@@&&(**#*(*#%#&@%%&%#&@@@@&@&&&@@@@@@@@@@@%%%%#//(*/#/#(%%%
|
|
#%#%&#&%%*,/##&@@@@@@&&&&&#(*#//#%&&&%&(/&@&@@@@@@&&&&@@@@@@@@@&%(%%(%(##%*//(%&
|
|
&&%(**/%&&&%&&@&@&@@@%&&@@@#####%&&@##//&%%%&@&&@&@@&@&&@@@@@@&&&%#/##(%##%&&&&#
|
|
**(#&%&%%&&%&&@@@@@@@@@@@@@@&%&%@@&&#%(@&&&&%&&&&@@@&&&@@@@@@@@/*%%%&&&&&%%%%##/
|
|
%&&%&%/***%&@@@&&@@@@@@&&@&&&@&&@@@%(%&&&&%%&&%&&&%@@@@@@@@@@@@&&&@&&%/,***,***/
|
|
|
|
The limits of my language means the limits of my world
|
|
|
|
- Ludwig Wittgenstein
|
|
"""
|
|
|
|
|
|
st.title("Unsupervised Extractive Text Summarization and Semantic Search - by Allen Roush!")
|
|
st.caption("Under active development! Please contact me or drop an issue on https://github.com/Hellisotherpeople/CX_DB8")
|
|
|
|
with st.expander("Dedicated to the late..."):
|
|
st.text(wit_text)
|
|
|
|
|
|
model_name = st.text_area("Enter the name of the pre-trained model from sentence transformers that we are using for summarization", value = "paraphrase-MiniLM-L3-v2")
|
|
st.caption("This will download a new model, so it may take awhile or even break if the model is too large")
|
|
st.caption("See the list of pre-trained models that are available here! https://www.sbert.net/docs/pretrained_models.html")
|
|
embedder = SentenceTransformer(model_name)
|
|
|
|
|
|
granularity = st.radio("What level of granularity do you want to summarize at?", ('Sentence', 'Word', 'Paragraph'))
|
|
|
|
y = st.text_area('enter a query', value = "I love chocolate")
|
|
x = st.text_area('Enter a document', value = choclate_example)
|
|
percentage = st.number_input("Enter the percentage of the text you want highlighted", max_value = 0.99, min_value = 0.01, value = 0.3)
|
|
|
|
query_embedding = embedder.encode(y, convert_to_tensor=True)
|
|
|
|
if granularity == "Sentence":
|
|
doc = nlp(x)
|
|
doc_sents = [str(sent) for sent in doc.sents]
|
|
corpus_embeddings = embedder.encode(doc_sents, convert_to_tensor=True)
|
|
len_doc = len(doc_sents)
|
|
elif granularity == "Word":
|
|
split_words = x.split()
|
|
ww_size = st.number_input("What size do you want the word window to be?", value = 3, min_value = 1, max_value=int(len(split_words)/4))
|
|
window_words = list(more_itertools.windowed(split_words, ww_size))
|
|
len_doc = len(window_words)
|
|
corpus_embeddings = embedder.encode(window_words, convert_to_tensor=True)
|
|
elif granularity == "Paragraph":
|
|
split_lines = x.splitlines()
|
|
corpus_embeddings = embedder.encode(split_lines, convert_to_tensor=True)
|
|
len_doc = len(split_lines)
|
|
|
|
semantic_search_results = util.semantic_search(query_embedding, corpus_embeddings, top_k= int(percentage * len_doc))
|
|
|
|
list_of_selected_extracts = []
|
|
for extract in semantic_search_results[0]:
|
|
list_of_selected_extracts.append(extract["corpus_id"])
|
|
|
|
if granularity == "Sentence":
|
|
original_granularity = doc_sents
|
|
elif granularity == "Word":
|
|
original_granularity = split_words
|
|
elif granularity == "Paragraph":
|
|
original_granularity = split_lines
|
|
|
|
string_to_print = ""
|
|
for count, chunk in enumerate(original_granularity):
|
|
if count in list_of_selected_extracts:
|
|
string_to_add = "\u0332".join(" " + chunk)
|
|
string_to_print += string_to_add
|
|
else:
|
|
string_to_print += " "
|
|
string_to_print += chunk
|
|
|
|
st.subheader("Output summary")
|
|
st.write(string_to_print)
|
|
|
|
st.subheader("Raw results")
|
|
st.caption("corpus_id is the number of the word, sentence, or paragraph. Score is the raw cosine similarty score between the document and the query")
|
|
st.write(semantic_search_results[0])
|
|
|
|
st.subheader("Results of segmentation/tokenization")
|
|
st.caption("This shows the representation that the webapp gets of the input document. Useful for debugging if you get strange output")
|
|
|
|
if granularity == "Sentence":
|
|
st.write(doc_sents)
|
|
elif granularity == "Word":
|
|
st.write(window_words)
|
|
elif granularity == "Paragraph":
|
|
st.write(split_lines) |
