Add dependency comp general functionality, fix issues and add more examples

app.py

@@ -8,7 +8,9 @@ from bs4 import BeautifulSoup
 import numpy as np
 import base64
 
+import validators
 from spacy_streamlit.util import get_svg
+from validators import ValidationFailure
 
 from custom_renderer import render_sentence_custom
 from flair.data import Sentence
@@ -134,6 +136,18 @@ def fetch_summary_contents(filename: str) -> AnyStr:
     return data
 
 
+def fetch_entity_specific_contents(filename: str) -> AnyStr:
+    with open(f'./entity-specific-text/{filename.lower()}.txt', 'r') as f:
+        data = f.read()
+    return data
+
+
+def fetch_dependency_specific_contents(filename: str) -> AnyStr:
+    with open(f'./dependency-specific-text/{filename.lower()}.txt', 'r') as f:
+        data = f.read()
+    return data
+
+
 def classify_comment(comment, selected_model):
     """Classify the given comment and augment with additional information."""
     toxicity_pipeline, cls_explainer = load_pipeline(selected_model)
@@ -162,9 +176,10 @@ def classify_comment(comment, selected_model):
 
 def display_summary(article_name: str):
     summary_content = fetch_summary_contents(article_name)
+    st.session_state.summary_output = summary_content
     soup = BeautifulSoup(summary_content, features="html.parser")
     HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem">{}</div>"""
-    st.session_state.summary_output = HTML_WRAPPER.format(soup)
+    return HTML_WRAPPER.format(soup)
 
 
 ##@st.cache(hash_funcs={preshed.maps.PreshMap: my_hash_func})
@@ -215,12 +230,12 @@ def get_all_entities(text):
 def get_and_compare_entities(article_name: str):
     article_content = fetch_article_contents(article_name)
     all_entities_per_sentence = get_all_entities_per_sentence(article_content)
-    #st.session_state.entities_per_sentence_article = all_entities_per_sentence
+    # st.session_state.entities_per_sentence_article = all_entities_per_sentence
     entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
     summary_content = fetch_summary_contents(article_name)
     all_entities_per_sentence = get_all_entities_per_sentence(summary_content)
-    #st.session_state.entities_per_sentence_summary = all_entities_per_sentence
+    # st.session_state.entities_per_sentence_summary = all_entities_per_sentence
     entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))
 
     matched_entities = []
@@ -268,45 +283,55 @@ def check_dependency(article: bool):
     if article:
         text = st.session_state.article_text
         all_entities = get_all_entities_per_sentence(text)
-        #all_entities = st.session_state.entities_per_sentence_article
+        # all_entities = st.session_state.entities_per_sentence_article
     else:
         text = st.session_state.summary_output
         all_entities = get_all_entities_per_sentence(text)
-        #all_entities = st.session_state.entities_per_sentence_summary
+        # all_entities = st.session_state.entities_per_sentence_summary
     doc = nlp(text)
     tok_l = doc.to_json()['tokens']
-    all_deps = ""
-    print(str(all_deps))
-    print("OOPS")
+    # all_deps = ""
+    test_list_dict_output = []
 
     sentences = list(doc.sents)
-    print(sentences)
     for i, sentence in enumerate(sentences):
-        #TODO MONDAY: THE PROBLEM LIES HERE WITH THE SENTENCE!!! (I THINK I KNOW PROBLEM: TEXT SAVED AS SESSION STATE IS HTML NOT PURE TEXT!)
-        print(str(sentence))
         start_id = sentence.start
         end_id = sentence.end
         for t in tok_l:
+            # print(t)
             if t["id"] < start_id or t["id"] > end_id:
                 continue
             head = tok_l[t['head']]
-            if t['dep'] == 'amod':
-                print("AMOD FOUND")
+            if t['dep'] == 'amod' or t['dep'] == "pobj":
                 object_here = text[t['start']:t['end']]
                 object_target = text[head['start']:head['end']]
-
+                if t['dep'] == "pobj" and str.lower(object_target) != "in":
+                    continue
                 # ONE NEEDS TO BE ENTITY
                 if object_here in all_entities[i]:
-                    print("SENTENCE ADDED")
-                    print(all_deps)
-                    all_deps = all_deps.join(str(sentence))
+                    # all_deps = all_deps.join(str(sentence))
+                    identifier = object_here + t['dep'] + object_target
+                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),
+                                                  "target_word_index": (t['head'] - sentence.start),
+                                                  "identifier": identifier, "sentence": str(sentence)})
                 elif object_target in all_entities[i]:
-                    all_deps = all_deps.join(str(sentence))
+                    # all_deps = all_deps.join(str(sentence))
+                    identifier = object_here + t['dep'] + object_target
+                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),
+                                                  "target_word_index": (t['head'] - sentence.start),
+                                                  "identifier": identifier, "sentence": str(sentence)})
                 else:
                     continue
-    #print(f'all depps are {all_deps}')
-    #print(all_deps)
-    return all_deps
+    # print(f'NOW TEST LIST DICT: {test_list_dict_output}')
+    return test_list_dict_output
+    # return all_deps
+
+
+def is_valid_url(url: str) -> bool:
+    result = validators.url(url)
+    if isinstance(result, ValidationFailure):
+        return False
+    return True
 
 
 # Start session
@@ -359,13 +384,28 @@ if st.session_state.article_text:
     with st.spinner('Generating summary...'):
         # classify_comment(article_text, selected_model)
 
-        display_summary(selected_article)
+        summary_displayed = display_summary(selected_article)
 
-        st.write("**Generated summary:**", st.session_state.summary_output, unsafe_allow_html=True)
+        st.write("**Generated summary:**", summary_displayed, unsafe_allow_html=True)
 else:
     st.error('**Error**: No comment to classify. Please provide a comment.',
              help="Generate summary for the given article text")
 
+if is_valid_url(article_text):
+    print("YES")
+else:
+    print("NO")
+def render_svg(svg_file):
+    with open(svg_file, "r") as f:
+        lines = f.readlines()
+        svg = "".join(lines)
+
+        # """Renders the given svg string."""
+        b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
+        html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64
+        return html
+
+
 # ENTITY MATCHING PART
 st.header("Entity matching")
 st.markdown("**Named entity recognition** (NER) is the task of identifying and categorising key information ("
@@ -376,23 +416,67 @@ st.markdown("**Named entity recognition** (NER) is the task of identifying and c
 with st.spinner("Calculating and matching entities..."):
     entity_match_html = highlight_entities(selected_article)
     st.write(entity_match_html, unsafe_allow_html=True)
+    red_text = """<font color="black"><span style="background-color: rgb(238, 135, 135); opacity: 
+    1;">red</span></font> """
+    green_text = """<font color="black">
+        <span style="background-color: rgb(121, 236, 121); opacity: 1;">green</span>
+    </font>"""
+
+    markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
+    markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
+    st.markdown("Here you can see what this looks like when we apply entity-matching on the summary (compared to the "
+                "original article). Entities in this summary are marked  " + green_text + " when the entity also "
+                                                                                          "exists in the article, while unmatched entities are marked " + red_text + ".",
+                unsafe_allow_html=True)
+    entity_specific_text = fetch_entity_specific_contents(selected_article)
+    st.markdown(entity_specific_text)
 
 # DEPENDENCY PARSING PART
 st.header("Dependency comparison")
+st.markdown("**Dependency parsing** is the process in which the grammatical structure in a sentence is analysed, "
+            "to find out related words as well as the type of the relationship between them. For the sentence “Jan’s "
+            "wife is called Sarah” you would get the following dependency graph:")
+
+# TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
+# st.image("ExampleParsing.svg")
+st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)
+st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
+            "husband…”, there would be a dependency in the summary that is non-existent in the article itself. "
+            "However, it could be that such a new dependency is not per se correct, “The borders of Ukraine” have a "
+            "different dependency between “borders” and “Ukraine” than “Ukraine’s borders”, while this would also be "
+            "correct. So general matching between summary and article wont work.")
+st.markdown("There is however a simple method that we found has potential in post-processing. Based on empirical "
+            "results, we have found that when there are specific kinds of dependencies in the summary that are not in "
+            "the article, these specific types are often an indication of a wrongly constructed sentence. Let’s take "
+            "a look at an example:")
 with st.spinner("Doing dependency parsing..."):
-    render_dependency_parsing(check_dependency(False))
-# Results
-# if 'results' in st.session_state and st.session_state.results:
-#     first = True
-#     for result in st.session_state.results[::-1]:
-#         if not first:
-#             st.markdown("---")
-#         st.markdown(f"Text:\n> {result['text']}")
-#         col_1, col_2, col_3 = st.columns([1,2,2])
-#         col_1.metric(label='', value=f"{result['emoji']}")
-#         col_2.metric(label='Label', value=f"{result['label']}")
-#         col_3.metric(label='Score', value=f"{result['score']:.3f}")
-#         st.markdown(f"Token Attribution:\n{result['tokens_with_background']}",
-#          unsafe_allow_html=True)
-#         st.caption(f"Model: {result['model_name']}")
-#         first = False
+    summary_deps = check_dependency(False)
+    article_deps = check_dependency(True)
+    total_unmatched_deps = []
+    for summ_dep in summary_deps:
+        if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+            total_unmatched_deps.append(summ_dep)
+    # print(f'ALL UNMATCHED DEPS ARE: {total_unmatched_deps}')
+    # render_dependency_parsing(check_dependency(False))
+    if total_unmatched_deps:
+        for current_drawing_list in total_unmatched_deps:
+            render_dependency_parsing(current_drawing_list)
+    dep_spec_text = fetch_dependency_specific_contents(selected_article)
+    st.markdown(dep_spec_text)
+    soup = BeautifulSoup("Example text option with box", features="html.parser")
+    HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;
+    margin-bottom: 2.5rem">{}</div> """
+    st.write(HTML_WRAPPER.format(soup), unsafe_allow_html=True)
+
+# OUTRO/CONCLUSION
+st.header("Wrapping up")
+st.markdown("We have presented 2 methods that try to improve summaries via post-processing steps. Entity matching can "
+            "be used to solve hallucinations, while checking if specific dependencies are matched between summary and "
+            "article can be used to filter out some bad sentences (and thus worse summaries). Of course these are "
+            "only basic methods which were empirically tested, but they are a start at actually making something good "
+            "(???). (something about that we tested also RE and maybe other things).")
+st.markdown("####")
+st.markdown("Now based on these methods you can check summaries and whether they are “good” or “bad”. Below you can "
+            "generate 5 different kind of summaries for the starting article (based on different model params) in "
+            "which their ranks are estimated, and hopefully the best summary (read: the one that a human would prefer "
+            "or indicate as the best one) will be at the top.")

custom_renderer.py

@@ -1,10 +1,12 @@
 from typing import Dict, Any
 
+import numpy as np
 import spacy
 from PIL import ImageFont
 
 from spacy.tokens import Doc
 
+
 def get_pil_text_size(text, font_size, font_name):
     font = ImageFont.truetype(font_name, font_size)
     size = font.getsize(text)
@@ -32,8 +34,8 @@ def render_arrow(
         <path class="displacy-arrowhead" d="{head}" fill="red"/>
     </g>
     """
-    arc = get_arc(start + 20, 50, 5, end + 20)
-    arrowhead = get_arrowhead(direction, start + 20, 50, end + 20)
+    arc = get_arc(start + 10, 50, 5, end + 10)
+    arrowhead = get_arrowhead(direction, start + 10, 50, end + 10)
     label_side = "right" if direction == "rtl" else "left"
     return TPL_DEP_ARCS.format(
         id=0,
@@ -77,7 +79,7 @@ def get_arrowhead(direction: str, x: int, y: int, end: int) -> str:
 
 
 # parsed = [{'words': [{'text': 'The', 'tag': 'DET', 'lemma': None}, {'text': 'OnePlus', 'tag': 'PROPN', 'lemma': None}, {'text': '10', 'tag': 'NUM', 'lemma': None}, {'text': 'Pro', 'tag': 'PROPN', 'lemma': None}, {'text': 'is', 'tag': 'AUX', 'lemma': None}, {'text': 'the', 'tag': 'DET', 'lemma': None}, {'text': 'company', 'tag': 'NOUN', 'lemma': None}, {'text': "'s", 'tag': 'PART', 'lemma': None}, {'text': 'first', 'tag': 'ADJ', 'lemma': None}, {'text': 'flagship', 'tag': 'NOUN', 'lemma': None}, {'text': 'phone.', 'tag': 'NOUN', 'lemma': None}], 'arcs': [{'start': 0, 'end': 3, 'label': 'det', 'dir': 'left'}, {'start': 1, 'end': 3, 'label': 'nmod', 'dir': 'left'}, {'start': 1, 'end': 2, 'label': 'nummod', 'dir': 'right'}, {'start': 3, 'end': 4, 'label': 'nsubj', 'dir': 'left'}, {'start': 5, 'end': 6, 'label': 'det', 'dir': 'left'}, {'start': 6, 'end': 10, 'label': 'poss', 'dir': 'left'}, {'start': 6, 'end': 7, 'label': 'case', 'dir': 'right'}, {'start': 8, 'end': 10, 'label': 'amod', 'dir': 'left'}, {'start': 9, 'end': 10, 'label': 'compound', 'dir': 'left'}, {'start': 4, 'end': 10, 'label': 'attr', 'dir': 'right'}], 'settings': {'lang': 'en', 'direction': 'ltr'}}]
-def render_sentence_custom(parsed: str):
+def render_sentence_custom(unmatched_list: Dict):
     TPL_DEP_WORDS = """
   <text class="displacy-token" fill="currentColor" text-anchor="start" y="{y}">
       <tspan class="displacy-word" fill="currentColor" x="{x}">{text}</tspan>
@@ -89,43 +91,94 @@ def render_sentence_custom(parsed: str):
   <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:lang="{lang}" id="{id}" class="displacy" width="{width}" height="{height}" direction="{dir}" style="max-width: none; height: {height}px; color: {color}; background: {bg}; font-family: {font}; direction: {dir}">{content}</svg>
   """
     arcs_svg = []
-    couples = []
-    nlp = spacy.load('en_core_web_sm')
-    doc = nlp(parsed)
-    arcs = {}
-    words = {}
-    parsed = [parse_deps(doc)]
-    for i, p in enumerate(parsed):
-        arcs = p["arcs"]
-        words = p["words"]
-    for i, a in enumerate(arcs):
-        if a["label"] == "amod":
-            couples = (a["start"], a["end"])
+    nlp = spacy.load('en_core_web_lg')
+    doc = nlp(unmatched_list["sentence"])
+    # words = {}
+    # unmatched_list = [parse_deps(doc)]
+    # #print(parsed)
+    # for i, p in enumerate(unmatched_list):
+    #     arcs = p["arcs"]
+    #     words = p["words"]
+    # for i, a in enumerate(arcs):
+    #     #CHECK CERTAIN DEPS (ALSO ADD/CHANGE BELOW WHEN CHANGING HERE)
+    #     if a["label"] == "amod":
+    #         couples = (a["start"], a["end"])
+    #     elif a["label"] == "pobj":
+    #         couples = (a["start"], a["end"])
+    # #couples = (3,5)
+    #
+    # x_value_counter = 10
+    # index_counter = 0
+    # svg_words = []
+    # coords_test = []
+    # for i, word in enumerate(words):
+    #     word = word["text"]
+    #     word = word + " "
+    #     pixel_x_length = get_pil_text_size(word, 16, 'arial.ttf')[0]
+    #     svg_words.append(TPL_DEP_WORDS.format(text=word, tag="", x=x_value_counter, y=70))
+    #     if index_counter >= couples[0] and index_counter <= couples[1]:
+    #         coords_test.append(x_value_counter)
+    #         x_value_counter += 50
+    #     index_counter += 1
+    #     x_value_counter += pixel_x_length + 4
+    # for i, a in enumerate(arcs):
+    #     if a["label"] == "amod":
+    #         arcs_svg.append(render_arrow(a["label"], coords_test[0], coords_test[-1], a["dir"], i))
+    #     elif a["label"] == "pobj":
+    #         arcs_svg.append(render_arrow(a["label"], coords_test[0], coords_test[-1], a["dir"], i))
+    #
+    # content = "".join(svg_words) + "".join(arcs_svg)
+    #
+    # full_svg = TPL_DEP_SVG.format(
+    #     id=0,
+    #     width=1200, #600
+    #     height=250, #125
+    #     color="#00000",
+    #     bg="#ffffff",
+    #     font="Arial",
+    #     content=content,
+    #     dir="ltr",
+    #     lang="en",
+    # )
 
     x_value_counter = 10
     index_counter = 0
     svg_words = []
+    words = unmatched_list["sentence"].split(" ")
     coords_test = []
-    for i, word in enumerate(words):
-        word = word["text"]
+    #print(unmatched_list)
+    #print(words)
+    #print("NOW")
+    direction_current = "rtl"
+    if unmatched_list["cur_word_index"] < unmatched_list["target_word_index"]:
+        min_index = unmatched_list["cur_word_index"]
+        max_index = unmatched_list["target_word_index"]
+        direction_current = "left"
+    else:
+        max_index = unmatched_list["cur_word_index"]
+        min_index = unmatched_list["target_word_index"]
+    for i, token in enumerate(doc):
+        word = str(token)
         word = word + " "
         pixel_x_length = get_pil_text_size(word, 16, 'arial.ttf')[0]
         svg_words.append(TPL_DEP_WORDS.format(text=word, tag="", x=x_value_counter, y=70))
-        if index_counter >= couples[0] and index_counter <= couples[1]:
+        if min_index <= index_counter <= max_index:
             coords_test.append(x_value_counter)
-            x_value_counter += 50
+            if index_counter < max_index - 1:
+                x_value_counter += 50
         index_counter += 1
         x_value_counter += pixel_x_length + 4
-    for i, a in enumerate(arcs):
-        if a["label"] == "amod":
-            arcs_svg.append(render_arrow(a["label"], coords_test[0], coords_test[-1], a["dir"], i))
+
+    # TODO: DYNAMIC DIRECTION MAKING (SHOULD GIVE WITH DICT I THINK)
+    #print(coords_test)
+    arcs_svg.append(render_arrow(unmatched_list['dep'], coords_test[0], coords_test[-1], direction_current, i))
 
     content = "".join(svg_words) + "".join(arcs_svg)
 
     full_svg = TPL_DEP_SVG.format(
         id=0,
-        width=1975,
-        height=574.5,
+        width=1200,  # 600
+        height=75,  # 125
         color="#00000",
         bg="#ffffff",
         font="Arial",
@@ -133,9 +186,9 @@ def render_sentence_custom(parsed: str):
         dir="ltr",
         lang="en",
     )
-
     return full_svg
 
+
 def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
     """Generate dependency parse in {'words': [], 'arcs': []} format.
 
@@ -196,8 +249,9 @@ def parse_deps(orig_doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]:
             )
     return {"words": words, "arcs": arcs, "settings": get_doc_settings(orig_doc)}
 
+
 def get_doc_settings(doc: Doc) -> Dict[str, Any]:
     return {
         "lang": doc.lang_,
         "direction": doc.vocab.writing_system.get("direction", "ltr"),
-    }
+    }

dependency-specific-text/article11.txt

@@ -0,0 +1,4 @@
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "poss" (possession modifier) dependency.
+In the image above, you can see the unmatched dependency that is found in the summary but not present in the article. For the "poss" dependency, we only check matches when the target word is "in", as it is here. U.S. is the entity here.
+For this specific example, it's obvious that the dependency of "in U.S." is not found in the article, as you can already see in the entity matching paragraph that U.S. is a hallucinated entity and doesn't occur in the article itself,
+so technically we don't need dependency comparison here to spot this particular error.

dependency-specific-text/article13.txt

@@ -0,0 +1,3 @@
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "amod" (adjectival modifier) dependency.
+In the image above, you can see the unmatched dependency that is found in the summary but not present in the article. "First" is the entity here, and it's the adjectival modifier of the word "phone".
+However, this sentence is not factual, since the article talks about a **new** type of flagship phone, and not at all the **first** flagship phone. This is wrong, and the error was found by filtering on this specific kind of dependency.

dependency-specific-text/example.txt

@@ -0,0 +1 @@
+This is example explanation.

entity-specific-text/article11.txt

@@ -0,0 +1,3 @@
+For this summary, there are 2 unmatched entities: "The Mark Levinson" and "U.S". The first one
+is not actually a real error per se, but rather a "the" before "Mark Levinson" (TODO EXPLAIN BIT BETTER).
+The "U.S." however is a hallucinated entity not present in the article, and via this method this can be found.

entity-specific-text/article13.txt

@@ -0,0 +1,2 @@
+For this summary, there are 2 unmatched entities: "January 18" and "U.S". January 18 is indeed a hallucinated entity, as there is no sentence containing this exact date. U.S. does occur in the article, but as "US" instead of "U.S.". This can be solved
+by comparing to a list of abbreviations (of embeddings :TODO?)

sample-articles/article11.txt

@@ -0,0 +1,13 @@
+Not so long ago, the internet was debating what the folks at Apple were thinking charging $549 for the AirPods Max. At CES 2022, luxury audio brand Mark Levinson would like a word. The Harman-owned company (which also owns AKG, JBL and Harman Kardon, and is itself a Samsung subsidiary) mostly known for its high-end home and car audio systems has announced its first wireless headphones: the No. 5909. While they offer everything you'd expect from a premium set, they have a nearly $1,000 price tag that only a select few might commit to.
+
+The over-ear No. 5909 packs 40mm Beryllium drivers "expertly tuned to the Harman curve." The company explains that "the Harman curve" is acoustic response that it says has taken decades of research to construct. The result here is "incredible acoustic performance" in a set of "reference class" wireless headphones. Mark Levinson says that audio performance meets the guidelines for Hi-Res Audio certification thanks to 24-bit/96kHz signal processing and 40kHz acoustic response. The No. 5909 supports LDAC, AAC and aptX Adaptive wireless codecs via Bluetooth 5.1.
+
+Mark Levinson promises you'll hear details you haven't before, like "the slightest breath an artist takes" or "a hidden harmony." The company explains that the same "world-class sound engineers" that built the luxury brand's amps, turntables and streaming players are behind the tuning of the ultra pricey No. 5909.
+
+Mark Levinson/Harman
+
+Sound quality isn't the only consideration though. The No. 5909 has adaptive active noise cancellation (ANC) with three modes "for premium sound isolation" and an Ambient Aware feature that lets you tune into your surroundings as needed. The company also packed in four microphones for calls that are equipped with a so-called Smart Wind Adaption feature. The materials used to make the headphones are also better than the mostly plastic sets we typically see. The No. 5909 is built with an aluminum frame, painted metallic earcups, leather headband and replaceable leather ear cushions. An included hard shell travel case comes stocked with a USB-C charging cable, USB-C to USB-A adaptor, two USB-C to 3.5mm cables, 3.5mm to 6.3mm adaptor, airplane adaptor and a polishing cloth. Basically, it's everything you'd need to use the headphones on any setup — wired, wireless or while traveling.
+
+Mark Levinson says you can expect up to 30 hours of use with adaptive ANC active and up to 34 hours with the feature disabled. A quick-charge feature will give you up to six hours of play time in 15 minutes. Via an app for Android and iOS, you'll get some control over the headphones, but the company didn't go into specifics there.
+
+The No. 5909 will be available in black, pewter and red color options starting today for $999.

sample-summaries/article11.txt

@@ -0,0 +1 @@
+The Mark Levinson No. 5909 is the company's first wireless headphones. It's "reference class" and comes in black, pewter and red color options.. The headphones start at $999 and will be available starting today in the U.S. A quick-charge feature will give you up to six hours of play time in 15 minutes, the company says, via an app for Android and iOS.The company also packed in four microphones for calls that are equipped with a so-called Smart Wind Adaption feature., via Bluetooth 5.1.

sample-summaries/article13.txt

@@ -1 +1 @@
-The OnePlus 10 Pro is the company's first flagship phone. It's the result of a merger between OnePlus and Oppo, which will be called "SuperVOOC" The phone is launching in China first on January 11. There's also no word on a US release date yet. The 10 Pro will have a 6.7-inch display and three cameras on the back. We don't have a price yet, but OnePlus' flagship prices have gone up every year so far, and the 9 Pro was $969.The phone will go on sale January 11 in China and January 18 in the U.S.
+The OnePlus 10 Pro is the company's first flagship phone. It's the result of a merger between OnePlus and Oppo, which will be called "SuperVOOC" The phone is launching in China first on January 11. There's also no word on a US release date yet. The 10 Pro will have a 6.7-inch display and three cameras on the back. We don't have a price yet, but OnePlus' flagship prices have gone up every year so far, and the 9 Pro was $969. The phone will go on sale January 11 in China and January 18 in the U.S.