Change article titles to actual names, change some text

app.py

@@ -22,6 +22,7 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; 
 margin-bottom: 2.5rem">{}</div> """
 
+
 @st.experimental_singleton
 def get_sentence_embedding_model():
     return SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
@@ -76,31 +77,31 @@ def list_all_article_names() -> list:
 def fetch_article_contents(filename: str) -> AnyStr:
     if filename == "Provide your own input":
         return " "
-    with open(f'./sample-articles/{filename.lower()}.txt', 'r') as f:
+    with open(f'./sample-articles/{filename}.txt', 'r') as f:
         data = f.read()
     return data
 
 
 def fetch_summary_contents(filename: str) -> AnyStr:
-    with open(f'./sample-summaries/{filename.lower()}.txt', 'r') as f:
+    with open(f'./sample-summaries/{filename}.txt', 'r') as f:
         data = f.read()
     return data
 
 
 def fetch_entity_specific_contents(filename: str) -> AnyStr:
-    with open(f'./entity-specific-text/{filename.lower()}.txt', 'r') as f:
+    with open(f'./entity-specific-text/{filename}.txt', 'r') as f:
         data = f.read()
     return data
 
 
 def fetch_dependency_specific_contents(filename: str) -> AnyStr:
-    with open(f'./dependency-specific-text/{filename.lower()}.txt', 'r') as f:
+    with open(f'./dependency-specific-text/{filename}.txt', 'r') as f:
         data = f.read()
     return data
 
 
 def fetch_dependency_svg(filename: str) -> AnyStr:
-    with open(f'./dependency-images/{filename.lower()}.txt', 'r') as f:
+    with open(f'./dependency-images/{filename}.txt', 'r') as f:
         lines = [line.rstrip() for line in f]
     return lines
 
@@ -145,10 +146,14 @@ def get_all_entities(text):
     return list(itertools.chain.from_iterable(all_entities_per_sentence))
 
 
-def get_and_compare_entities():
-    article_content = st.session_state.article_text
-    all_entities_per_sentence = get_all_entities_per_sentence(article_content)
-    entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
+def get_and_compare_entities(first_time: bool):
+    if first_time:
+        article_content = st.session_state.article_text
+        all_entities_per_sentence = get_all_entities_per_sentence(article_content)
+        entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
+        st.session_state.entities_article = entities_article
+    else:
+        entities_article = st.session_state.entities_article
 
     summary_content = st.session_state.summary_output
     all_entities_per_sentence = get_all_entities_per_sentence(summary_content)
@@ -166,6 +171,18 @@ def get_and_compare_entities():
             matched_entities.append(entity)
         else:
             unmatched_entities.append(entity)
+
+    matched_entities = list(dict.fromkeys(matched_entities))
+    unmatched_entities = list(dict.fromkeys(unmatched_entities))
+    for entity in matched_entities:
+        for substring_entity in matched_entities:
+            if entity != substring_entity and entity.lower() in substring_entity.lower():
+                matched_entities.remove(entity)
+
+    for entity in unmatched_entities:
+        for substring_entity in unmatched_entities:
+            if entity != substring_entity and entity.lower() in substring_entity.lower():
+                unmatched_entities.remove(entity)
     return matched_entities, unmatched_entities
 
 
@@ -175,7 +192,7 @@ def highlight_entities():
     markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
     markdown_end = "</mark>"
 
-    matched_entities, unmatched_entities = get_and_compare_entities()
+    matched_entities, unmatched_entities = get_and_compare_entities(True)
 
     for entity in matched_entities:
         summary_content = summary_content.replace(entity, markdown_start_green + entity + markdown_end)
@@ -263,32 +280,43 @@ def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs)
     return summary
 
 
-
-# Page
-st.title('📜 Summarization fact checker 📜')
-
-# INTRODUCTION
-st.header("Introduction")
-st.markdown("""Recent work using transformers on large text corpora has shown great success when fine-tuned on 
-several different downstream NLP tasks. One such task is that of text summarization. The goal of text summarization 
-is to generate concise and accurate summaries from input document(s). There are 2 types of summarization: extractive 
-and abstractive. **Extractive summarization** merely copies informative fragments from the input, 
-whereas **abstractive summarization** may generate novel words. A good abstractive summary should cover principal 
-information in the input and has to be linguistically fluent. This interactive blogpost will focus on this more difficult task of 
-abstractive summary generation.""")
-
-st.markdown("""To generate summaries we will use the [PEGASUS] (https://huggingface.co/google/pegasus-cnn_dailymail) 
-model, producing abstractive summaries from large articles. These summaries often contain sentences with different 
-kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to detect errors 
-from the generated summaries. Throughout this blog, we will also explain the results for some methods on specific 
-examples. These text blocks will be indicated and they change according to the currently selected article.""")
-
 # Load all different models (cached) at start time of the hugginface space
 sentence_embedding_model = get_sentence_embedding_model()
 ner_model = get_transformer_pipeline()
 nlp = get_spacy()
 summarization_model = get_summarizer_model()
 
+# Page
+st.title('📜 Summarization fact checker 📜')
+
+# INTRODUCTION
+st.header("🧑‍🏫 Introduction")
+
+introduction_checkbox = st.checkbox("Show introduction text", value = True)
+if introduction_checkbox:
+    st.markdown("""
+    Recent work using 🤖 **transformers** 🤖 on large text corpora has shown great success when fine-tuned on 
+    several different downstream NLP tasks. One such task is that of text summarization. The goal of text summarization 
+    is to generate concise and accurate summaries from input document(s). There are 2 types of summarization:
+    
+     - **Extractive summarization** merely copies informative fragments from the input
+     - **Abstractive summarization** may generate novel words. A good abstractive summary should cover principal 
+        information in the input and has to be linguistically fluent. This interactive blogpost will focus on this more difficult task of 
+        abstractive summary generation. Furthermore we will focus on factual errors in summaries, and less sentence fluency.""")
+
+    st.markdown("###")
+    st.markdown("🤔 **Why is this important?** 🤔 Let's say we want to summarize news articles for a popular "
+                "newspaper. If an article tells the story of **Putin** invading Ukraine, we don't want our summarization "
+                "model to say that **Biden** is invading Ukraine. Summarization could also be done for financial reports "
+                "for example. In such environments, these errors can be very critical, so we want to find a way to "
+                "detect them.")
+    st.markdown("###")
+    st.markdown("""To generate summaries we will use the 🐎 [PEGASUS](https://huggingface.co/google/pegasus-cnn_dailymail) 🐎
+    model, producing abstractive summaries from large articles. These summaries often contain sentences with different 
+    kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to detect errors 
+    from the generated summaries. Throughout this blog, we will also explain the results for some methods on specific 
+    examples. These text blocks will be indicated and they change according to the currently selected article.""")
+
 # GENERATING SUMMARIES PART
 st.header("🪶 Generating summaries")
 st.markdown("Let’s start by selecting an article text for which we want to generate a summary, or you can provide "
@@ -296,26 +324,28 @@ st.markdown("Let’s start by selecting an article text for which we want to gen
             "summary generated from it might not be optimal, leading to suboptimal performance of the post-processing "
             "steps. However, too long articles will be truncated and might miss information in the summary.")
 
+st.markdown("####")
 selected_article = st.selectbox('Select an article or provide your own:',
-                                list_all_article_names())
+                                list_all_article_names(), index=2)
 st.session_state.article_text = fetch_article_contents(selected_article)
 article_text = st.text_area(
     label='Full article text',
     value=st.session_state.article_text,
-    height=150
+    height=250
 )
 
-summarize_button = st.button(label='Process article content',
+summarize_button = st.button(label='🤯 Process article content',
                              help="Start interactive blogpost")
 
 if summarize_button:
     st.session_state.article_text = article_text
+    st.markdown("####")
     st.markdown(
-        "Below you can find the generated summary for the article. We will discuss two approaches that we found are "
+        "*Below you can find the generated summary for the article. We will discuss two approaches that we found are "
         "able to detect some common errors. Based on errors, one could then score different summaries, indicating how "
         "factual a summary is for a given article. The idea is that in production, you could generate a set of "
         "summaries for the same article, with different parameters (or even different models). By using "
-        "post-processing error detection, we can then select the best possible summary.")
+        "post-processing error detection, we can then select the best possible summary.*")
     if st.session_state.article_text:
         with st.spinner('Generating summary, this might take a while...'):
             if selected_article != "Provide your own input" and article_text == fetch_article_contents(
@@ -327,18 +357,25 @@ if summarize_button:
                                                                no_repeat_ngram_size=4)
                 st.session_state.unchanged_text = False
             summary_displayed = display_summary(summary_content)
-            st.write("**Generated summary:**", summary_displayed, unsafe_allow_html=True)
+            st.write("✍ **Generated summary:** ✍", summary_displayed, unsafe_allow_html=True)
     else:
         st.error('**Error**: No comment to classify. Please provide a comment.')
 
     # ENTITY MATCHING PART
-    st.header("Entity matching")
+    st.header("1️⃣ Entity matching")
     st.markdown("The first method we will discuss is called **Named Entity Recognition** (NER). NER is the task of "
                 "identifying and categorising key information (entities) in text. An entity can be a singular word or a "
                 "series of words that consistently refers to the same thing. Common entity classes are person names, "
                 "organisations, locations and so on. By applying NER to both the article and its summary, we can spot "
-                "possible **hallucinations**. Hallucinations are words generated by the model that are not supported by "
-                "the source input. In theory all entities in the summary (such as dates, locations and so on), "
+                "possible **hallucinations**. ")
+
+    st.markdown("Hallucinations are words generated by the model that are not supported by "
+                "the source input. Deep learning based generation is [prone to hallucinate]("
+                "https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade "
+                "system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem"
+                " for the downstream task of summary generation.")
+
+    st.markdown(" In theory all entities in the summary (such as dates, locations and so on), "
                 "should also be present in the article. Thus we can extract all entities from the summary and compare "
                 "them to the entities of the original article, spotting potential hallucinations. The more unmatched "
                 "entities we find, the lower the factualness score of the summary. ")
@@ -354,7 +391,7 @@ if summarize_button:
         markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
         markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
         st.markdown(
-            "We call this technique “entity matching” and here you can see what this looks like when we apply this "
+            "We call this technique **entity matching** and here you can see what this looks like when we apply this "
             "method on the summary. Entities in the summary are marked  " + green_text + " when the entity also "
                                                                                          "exists in the article, "
                                                                                          "while unmatched entities "
@@ -365,10 +402,11 @@ if summarize_button:
         if st.session_state.unchanged_text:
             entity_specific_text = fetch_entity_specific_contents(selected_article)
             soup = BeautifulSoup(entity_specific_text, features="html.parser")
+            st.markdown("####")
             st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)
 
     # DEPENDENCY PARSING PART
-    st.header("Dependency comparison")
+    st.header("2️⃣ Dependency comparison")
     st.markdown(
         "The second method we use for post-processing is called **Dependency parsing**: the process in which the "
         "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "
@@ -378,16 +416,23 @@ if summarize_button:
     # TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)
     # st.image("ExampleParsing.svg")
     st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)
-    st.markdown("Here, “Jan” is the “poss” (possession modifier) of “wife”. If suddenly the summary would read “Jan’s "
-                "husband…”, there would be a dependency in the summary that is non-existent in the article itself (namely "
-                "“Jan” is the “poss” of “husband”). However, often new dependencies are introduced in the summary that "
-                "are still correct. “The borders of Ukraine” have a different dependency between “borders” and "
-                "“Ukraine” "
-                "than “Ukraine’s borders”, while both descriptions have the same meaning. So just matching all "
-                "dependencies between article and summary (as we did with entity matching) would not be a robust method.")
-    st.markdown("However, we have found that there are specific dependencies that, when unmatched, are often an "
-                "indication of a wrongly constructed sentence. We found 2(/3 TODO) common dependencies which, "
-                "when present in the summary but not in the article, are highly indicative of factualness errors. "
+    st.markdown(
+        "Here, *“Jan”* is the *“poss”* (possession modifier) of *“wife”*. If suddenly the summary would read *“Jan’s"
+        " husband…”*, there would be a dependency in the summary that is non-existent in the article itself (namely "
+        "*“Jan”* is the “poss” of *“husband”*)."
+        "However, often new dependencies are introduced in the summary that "
+        "are still correct, as can be seen in the example below. ")
+    st.write(render_svg('SecondExampleParsing.svg'), unsafe_allow_html=True)
+
+    st.markdown("*“The borders of Ukraine”* have a different dependency between *“borders”* and "
+                "*“Ukraine”* "
+                "than *“Ukraine’s borders”*, while both descriptions have the same meaning. So just matching all "
+                "dependencies between article and summary (as we did with entity matching) would not be a robust method."
+                " More on the different sorts of dependencies and their description can be found [here](https://universaldependencies.org/docs/en/dep/).")
+    st.markdown("However, we have found that **there are specific dependencies that are often an "
+                "indication of a wrongly constructed sentence** -when there is no article match. We (currently) use 2 "
+                "common dependencies which - when present in the summary but not in the article - are highly "
+                "indicative of factualness errors. "
                 "Furthermore, we only check dependencies between an existing **entity** and its direct connections. "
                 "Below we highlight all unmatched dependencies that satisfy the discussed constraints. We also "
                 "discuss the specific results for the currently selected example article.")
@@ -410,14 +455,51 @@ if summarize_button:
                     render_dependency_parsing(current_drawing_list)
 
     # OUTRO/CONCLUSION
-    st.header("Wrapping up")
+    st.header("🤝 Bringing it together")
     st.markdown("We have presented 2 methods that try to detect errors in summaries via post-processing steps. Entity "
                 "matching can be used to solve hallucinations, while dependency comparison can be used to filter out "
                 "some bad sentences (and thus worse summaries). These methods highlight the possibilities of "
                 "post-processing AI-made summaries, but are only a first introduction. As the methods were "
                 "empirically tested they are definitely not sufficiently robust for general use-cases.")
     st.markdown("####")
-    st.markdown("(TODO) Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
-                "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
-                "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
-                "the actual parameters or something? ")
+    st.markdown(
+        "(TODO) Below we generated 5 different kind of summaries from the article in which their ranks are estimated, "
+        "and hopefully the best summary (read: the one that a human would prefer or indicate as the best one) "
+        "will be at the top. TODO: implement this (at the end I think) and also put something in the text with "
+        "the actual parameters or something? ")
+
+    # with st.spinner("Calculating more summaries and scoring them, might take while..."):
+    #     # ENTITIES
+    #     _, amount_unmatched = get_and_compare_entities(False)
+    #     st.write(len(amount_unmatched))
+    #     st.write(amount_unmatched)
+    #
+    #     # DEPS
+    #     summary_deps = check_dependency(False)
+    #     article_deps = check_dependency(True)
+    #     total_unmatched_deps = []
+    #     for summ_dep in summary_deps:
+    #         if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+    #             total_unmatched_deps.append(summ_dep)
+    #
+    #     st.write(len(total_unmatched_deps))
+    #     st.write(total_unmatched_deps)
+    #
+    #     # FOR NEW GENERATED SUMMARY
+    #     st.session_state.summary_output = generate_abstractive_summary(st.session_state.article_text,
+    #                                                                    type="beam",
+    #                                                                    do_sample=True, num_beams=15,
+    #                                                                    no_repeat_ngram_size=5)
+    #     _, amount_unmatched = get_and_compare_entities(False)
+    #     st.write(len(amount_unmatched))
+    #     st.write(amount_unmatched)
+    #
+    #     summary_deps = check_dependency(False)
+    #     article_deps = check_dependency(True)
+    #     total_unmatched_deps = []
+    #     for summ_dep in summary_deps:
+    #         if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+    #             total_unmatched_deps.append(summ_dep)
+    #
+    #     st.write(len(total_unmatched_deps))
+    #     st.write(total_unmatched_deps)

dependency-specific-text/{article11.txt → Mark Levinson Airpods.txt}

@@ -1,4 +1,4 @@
-One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "pobj" (object of preposition) dependency.
-Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
-In this case it's obvious that "in U.S." is not found in the article, as "U.S." is a hallucinated entity itself as discussed in the entity matching paragraph.
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the *"pobj"*(object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is *"in"*, as in this example.
+In this case it's obvious that *"in U.S."* is not found in the article, as *"U.S."* is a hallucinated entity itself as discussed in the entity matching paragraph.
 So technically we don't need dependency comparison to spot the error from this summary.

dependency-specific-text/{article9.txt → Novak Djokovic.txt}

@@ -1,4 +1,4 @@
-One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "pobj" (object of preposition) dependency.
-Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
-The sentence here is not a factual error per se, but rather a readability issue. The "in" should be dropped to make the sentence correct.
-For better examples with this specific dependency, try choosing another article. TODO: readability issue with the dependency graph for this specific issue
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the *"pobj"* (object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is *"in"*, as in this example.
+The sentence here is not a factual error per se, but rather a readability issue. The *"in"* should be dropped to make the sentence correct.
+For better examples with this specific dependency, try choosing another article.

dependency-specific-text/{article13.txt → OnePlus 10 Pro.txt}

@@ -1,9 +1,9 @@
-One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "amod" (adjectival modifier) dependency.
-Applied to this summary, we have "First" as the entity, and it is the adjectival modifier of the word "phone".
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the *"amod"* (adjectival modifier) dependency.
+Applied to this summary, we have *"First"* as the entity, and it is the adjectival modifier of the word *"phone"*.
 And indeed, this unmatched dependency indicates an actual error here. The sentence is not factual, since the article talks about a **new** type of flagship phone,
 and not the **first** flagship phone. This error was found by filtering on this specific kind of dependency. Empirical results showed that unmatched *amod* dependencies often suggest
 that the summary sentence contains an error.  <br> <br>
-Another dependency that we use is the "pobj" (object of preposition) dependency.
-Furthermore, we only match *pobj* dependencies when the target word is "in", as in this example.
-In this case the sentence itself contains a factual error (because the article states "there's no word on a US release date yet").
-However, this could have been found by entity matching already (as january 18 is unmatched), and the unmatched dependency can not be completely blamed for this error here.
+Another dependency that we use is the *"pobj"* (object of preposition) dependency.
+Furthermore, we only match *pobj* dependencies when the target word is *"in"*, as in this example.
+In this case the sentence itself contains a factual error (because the article states *"there's no word on a US release date yet"*).
+However, this could have already been found with entity matching (as *January 18* is unmatched), and the unmatched dependency can not be completely blamed for this error here.

dependency-specific-text/{article4.txt → White House.txt}

@@ -1,7 +1,7 @@
-One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the "amod" (adjectival modifier) dependency.
-Applied to this summary, we have "Democratic" as the entity, and it is the adjectival modifier of the word "member".
+One of the dependencies that, when found in the summary but not in the article, indicates a possible error is the *"amod"* (adjectival modifier) dependency.
+Applied to this summary, we have *"Democratic"* as the entity, and it is the adjectival modifier of the word *"member"*.
 And indeed, this unmatched dependency indicates an actual error here. The sentence is not factual for two reasons. <br> <br>
-First, the article talks about "democrats" and "members of the committee", which are two separate things. The summary combines those two in a way
+First, the article talks about *"democrats"* and *"members of the committee"*, which are two separate things. The summary combines those two in a way
 that can be seen as not completely factual. Second, the statement itself was not made by a democrat (nor a member of the committee), and even though the dependency can't be
 directly linked to this error, empirical results showed that unmatched *amod* dependencies often suggest
 that the summary sentence is incorrect.

entity-specific-text/Mark Levinson Airpods.txt

@@ -0,0 +1,2 @@
+As you can see we have 1 unmatched entity: *"U.S."* is a hallucinated entity in the summary, that does not exist in the article.
+This error can be found/solved by applying entity matching.

entity-specific-text/OnePlus 10 Pro.txt

@@ -0,0 +1,3 @@
+As you can see we have 2 unmatched entities: *"January 18"* and *"U.S"*. The first one is a hallucinated entity in the summary, that does not exist in the article.
+*U.S*. **does** occur in the article, but as *"US"* instead of *"U.S."*. This could be solved
+by comparing to a list of abbreviations or with a specific embedder for abbreviations but is currently not implemented.

entity-specific-text/Tencent Holdings.txt

@@ -0,0 +1,2 @@
+As you can see we have 1 unmatched entity: *"Six9"* is a hallucinated entity in the summary, that does not exist in the article.
+This error can be found/solved by applying entity matching.

entity-specific-text/article11.txt

@@ -1,4 +0,0 @@
-As you can see we have 1 unmatched entity: "U.S." is a hallucinated entity in the summary, that does not exist in the article.
-Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
-system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
-for the downstream task of summary generation.

entity-specific-text/article13.txt

@@ -1,5 +0,0 @@
-As you can see we have 2 unmatched entities: "January 18" and "U.S". The first one is a hallucinated entity in the summary, that does not exist in the article.
-Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
-system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
-for the downstream task of summary generation. U.S. **does** occur in the article, but as "US" instead of "U.S.". This could be solved
-by comparing to a list of abbreviations or with a specific embedder for abbreviations but is currently not implemented.

entity-specific-text/article16.txt

@@ -1,4 +0,0 @@
-As you can see we have 1 unmatched entity: "Six9" is a hallucinated entity in the summary, that does not exist in the article.
-Deep learning based generation is [prone to hallucinate](https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade
-system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem
-for the downstream task of summary generation.

sample-articles-temp/numbers_to_text.txt

@@ -0,0 +1,6 @@
+4: White House
+9: Novak Djokovic
+11: Mark Levinson Airpods
+13: OnePlus 10 Pro
+16: Tencent Holdings
+