Add app files

app.py

@@ -0,0 +1,281 @@
+import pandas as pd
+import gradio as gr
+from retrieval import retrieve_from_pdf
+
+if gr.NO_RELOAD:
+    from transformers import pipeline
+
+    # Keep track of the model name in a global variable so correct model is shown after page refresh
+    # https://github.com/gradio-app/gradio/issues/3173
+    MODEL_NAME = "jedick/DeBERTa-v3-base-mnli-fever-anli-scifact-citint"
+    pipe = pipeline(
+        "text-classification",
+        model=MODEL_NAME,
+    )
+
+
+def query_model(claim, evidence):
+    """
+    Get prediction for a pair of claim and evidence
+    """
+    prediction = {
+        # Send a dictionary containing {"text", "text_pair"} keys; use top_k=3 to get results for all classes
+        #   https://huggingface.co/docs/transformers/v4.51.3/en/main_classes/pipelines#transformers.TextClassificationPipeline.__call__.inputs
+        # Put evidence before claim
+        #   https://github.com/jedick/ML-capstone-project
+        # Output {label: confidence} dictionary format as expected by gr.Label()
+        #   https://github.com/gradio-app/gradio/issues/11170
+        d["label"]: d["score"]
+        for d in pipe({"text": evidence, "text_pair": claim}, top_k=3)
+    }
+    # Return two instances of the prediction to send to different Gradio components
+    return prediction, prediction
+
+
+def query_model_for_examples(claim, evidence):
+    """
+    A duplicate of the previous function, used to keep the API names clean
+    """
+    prediction = {
+        d["label"]: d["score"]
+        for d in pipe({"text": evidence, "text_pair": claim}, top_k=3)
+    }
+    return prediction, prediction
+
+
+# Function to select the model
+def select_model(model_name):
+    global pipe, MODEL_NAME
+    MODEL_NAME = model_name
+    pipe = pipeline(
+        "text-classification",
+        model=MODEL_NAME,
+    )
+
+
+def prediction_to_df(prediction=None):
+    """
+    Convert prediction text to DataFrame for barplot
+    """
+    if prediction is None or prediction == "":
+        # Show an empty plot for app initialization or auto-reload
+        prediction = {"SUPPORT": 0, "NEI": 0, "REFUTE": 0}
+    elif "Model" in prediction:
+        # Show full-height bars when the model is changed
+        prediction = {"SUPPORT": 1, "NEI": 1, "REFUTE": 1}
+    else:
+        # Convert predictions text to dictionary
+        prediction = eval(prediction)
+        # Rename dictionary keys to use consistent labels across models
+        prediction = {
+            ("SUPPORT" if k == "entailment" else k): v for k, v in prediction.items()
+        }
+        prediction = {
+            ("NEI" if k == "neutral" else k): v for k, v in prediction.items()
+        }
+        prediction = {
+            ("REFUTE" if k == "contradiction" else k): v for k, v in prediction.items()
+        }
+        # Use custom order for labels (pipe() returns labels in descending order of softmax score)
+        labels = ["SUPPORT", "NEI", "REFUTE"]
+        prediction = {k: prediction[k] for k in labels}
+    # Convert dictionary to DataFrame with one column (Probability)
+    df = pd.DataFrame.from_dict(prediction, orient="index", columns=["Probability"])
+    # Move the index to the Class column
+    return df.reset_index(names="Class")
+
+
+def change_visualization(choice):
+    if choice == "barplot":
+        barplot = gr.update(visible=True)
+        label = gr.update(visible=False)
+    elif choice == "label":
+        barplot = gr.update(visible=False)
+        label = gr.update(visible=True)
+    return barplot, label
+
+
+# Setup theme without background image
+my_theme = gr.Theme.from_hub("NoCrypt/miku")
+my_theme.set(body_background_fill="#FFFFFF", body_background_fill_dark="#000000")
+
+# Gradio interface setup
+with gr.Blocks(theme=my_theme) as demo:
+
+    # Layout
+    with gr.Row():
+        with gr.Column(scale=3):
+            with gr.Row():
+                gr.Markdown(
+                    """
+                # AI4citations
+                ## Scientific citation verification
+
+                *Press Enter in a textbox or click Submit to run the model.*
+                """
+                )
+                gr.Markdown(
+                    """
+                ### Three ways to use this app
+
+                1. **Claim verification**: Input a claim and evidence
+                2. **Evidence retrieval**: Input a claim to get evidence from PDF
+                3. **Claim extraction**: Input a text to get claim from text
+                """
+                )
+            # Create dropdown menu to select the model
+            dropdown = gr.Dropdown(
+                choices=[
+                    # TODO: For bert-base-uncased, how can we set num_labels = 2 in HF pipeline?
+                    # (num_labels is available in AutoModelForSequenceClassification.from_pretrained)
+                    # "bert-base-uncased",
+                    "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
+                    "jedick/DeBERTa-v3-base-mnli-fever-anli-scifact-citint",
+                ],
+                value=MODEL_NAME,
+                label="Model",
+            )
+            claim = gr.Textbox(
+                label="Claim",
+                info="aka hypothesis",
+                placeholder="Input claim or use Get Claim from Text",
+            )
+            evidence = gr.TextArea(
+                label="Evidence",
+                info="aka premise",
+                placeholder="Input evidence or use Get Evidence from PDF",
+            )
+            with gr.Row():
+                with gr.Accordion("Get Claim from Text", open=False):
+                    text = gr.TextArea(
+                        label="Text",
+                        placeholder="Under construction!",
+                        interactive=False,
+                    )
+                with gr.Accordion("Get Evidence from PDF", open=False):
+                    pdf_file = gr.File(label="Upload PDF", type="filepath")
+                    get_evidence = gr.Button(value="Get Evidence")
+                    top_k = gr.Slider(
+                        1,
+                        10,
+                        value=5,
+                        step=1,
+                        interactive=True,
+                        label="Top k sentences",
+                    )
+            submit = gr.Button("Submit")
+
+        with gr.Column(scale=2):
+            radio = gr.Radio(["barplot", "label"], value="barplot", label="Results")
+            # Keep the prediction textbox hidden
+            with gr.Accordion(visible=False):
+                prediction = gr.Textbox(label="Prediction")
+            barplot = gr.BarPlot(
+                prediction_to_df,
+                x="Class",
+                y="Probability",
+                color="Class",
+                color_map={"SUPPORT": "green", "NEI": "#888888", "REFUTE": "#FF8888"},
+                inputs=prediction,
+                y_lim=([0, 1]),
+            )
+            label = gr.Label(visible=False)
+            with gr.Accordion("Examples", open=False):
+                gr.Markdown(
+                    "*Prediction performance with jedick/DeBERTa-v3-base-mnli-fever-anli-scifact-citint:*"
+                ),
+                with gr.Row():
+                    gr.Examples(
+                        examples="examples/accurate",
+                        inputs=[claim, evidence],
+                        outputs=[prediction, label],
+                        fn=query_model_for_examples,
+                        label="Accurate",
+                        run_on_click=True,
+                        example_labels=pd.read_csv("examples/accurate/log.csv")[
+                            "label"
+                        ].tolist(),
+                    )
+                    gr.Examples(
+                        examples="examples/inaccurate",
+                        inputs=[claim, evidence],
+                        outputs=[prediction, label],
+                        fn=query_model_for_examples,
+                        label="Inaccurate",
+                        run_on_click=True,
+                        example_labels=pd.read_csv("examples/inaccurate/log.csv")[
+                            "label"
+                        ].tolist(),
+                    )
+                gr.Examples(
+                    examples="examples/retrieval",
+                    inputs=[pdf_file, claim],
+                    outputs=evidence,
+                    fn=retrieve_from_pdf,
+                    label="Retrieval",
+                    run_on_click=False,
+                    example_labels=pd.read_csv("examples/retrieval/log.csv")[
+                        "label"
+                    ].tolist(),
+                )
+            gr.Markdown(
+                """
+            ### Sources
+            - ML project: [jedick/ML-capstone-project](https://github.com/jedick/ML-capstone-project)
+                - App repository: [jedick/AI4citations](https://github.com/jedick/AI4citations)
+                - Fine-tuned model: [jedick/DeBERTa-v3-base-mnli-fever-anli-scifact-citint](https://huggingface.co/jedick/DeBERTa-v3-base-mnli-fever-anli-scifact-citint)
+            - Datasets used for fine-tuning
+                - SciFact: [allenai/SciFact](https://github.com/allenai/scifact)
+                - Citation-Integrity (CitInt): [ScienceNLP-Lab/Citation-Integrity](https://github.com/ScienceNLP-Lab/Citation-Integrity)
+            - Base model: [MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli](https://huggingface.co/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli)
+            - Evidence retrieval: [xhluca/bm25s](https://github.com/xhluca/bm25s)
+            - Gradio theme: [NoCrypt/miku](https://huggingface.co/spaces/NoCrypt/miku)
+            """
+            )
+
+    # Event listeners
+
+    # Click the submit button or press Enter to submit
+    gr.on(
+        triggers=[claim.submit, evidence.submit, submit.click],
+        fn=query_model,
+        inputs=[claim, evidence],
+        outputs=[prediction, label],
+    )
+
+    # Clear the previous predictions when the model is changed
+    gr.on(
+        triggers=[dropdown.select],
+        fn=lambda: "Model changed! Waiting for updated predictions...",
+        outputs=[prediction],
+        api_name=False,
+    )
+
+    # Update the predictions after changing the model
+    dropdown.change(
+        fn=select_model,
+        inputs=dropdown,
+    ).then(
+        fn=query_model,
+        inputs=[claim, evidence],
+        outputs=[prediction, label],
+        api_name=False,
+    )
+
+    # Get evidence from PDF
+    gr.on(
+        triggers=[pdf_file.upload, get_evidence.click],
+        fn=retrieve_from_pdf,
+        inputs=[pdf_file, claim, top_k],
+        outputs=evidence,
+    )
+
+    # Change visualization
+    radio.change(
+        fn=change_visualization,
+        inputs=radio,
+        outputs=[barplot, label],
+        api_name=False,
+    )
+
+demo.launch()

examples/accurate/log.csv

@@ -0,0 +1,7 @@
+claim,evidence,label
+"Poirot was now back and I was sorry that he would take over what I now considered my own investigation.","Poirot, I exclaimed, with relief, and seizing him by both hands, I dragged him into the room.","REFUTE (MNLI)"
+"0-dimensional biomaterials lack inductive properties.","Nanotechnologies are emerging platforms that could be useful in measuring, understanding, and manipulating stem cells. Examples include magnetic nanoparticles and quantum dots for stem cell labeling and in vivo tracking; nanoparticles, carbon nanotubes, and polyplexes for the intracellular delivery of genes/oligonucleotides and protein/peptides; and engineered nanometer-scale scaffolds for stem cell differentiation and transplantation. This review examines the use of nanotechnologies for stem cell tracking, differentiation, and transplantation. We further discuss their utility and the potential concerns regarding their cytotoxicity.","NEI (SciFact)"
+"1 in 5 million in UK have abnormal PrP positivity.","OBJECTIVES To carry out a further survey of archived appendix samples to understand better the differences between existing estimates of the prevalence of subclinical infection with prions after the bovine spongiform encephalopathy epizootic and to see whether a broader birth cohort was affected, and to understand better the implications for the management of blood and blood products and for the handling of surgical instruments. DESIGN Irreversibly unlinked and anonymised large scale survey of archived appendix samples. SETTING Archived appendix samples from the pathology departments of 41 UK hospitals participating in the earlier survey, and additional hospitals in regions with lower levels of participation in that survey. SAMPLE 32,441 archived appendix samples fixed in formalin and embedded in paraffin and tested for the presence of abnormal prion protein (PrP). RESULTS Of the 32,441 appendix samples 16 were positive for abnormal PrP, indicating an overall prevalence of 493 per million population (95% confidence interval 282 to 801 per million). The prevalence in those born in 1941-60 (733 per million, 269 to 1596 per million) did not differ significantly from those born between 1961 and 1985 (412 per million, 198 to 758 per million) and was similar in both sexes and across the three broad geographical areas sampled. Genetic testing of the positive specimens for the genotype at PRNP codon 129 revealed a high proportion that were valine homozygous compared with the frequency in the normal population, and in stark contrast with confirmed clinical cases of vCJD, all of which were methionine homozygous at PRNP codon 129. CONCLUSIONS This study corroborates previous studies and suggests a high prevalence of infection with abnormal PrP, indicating vCJD carrier status in the population compared with the 177 vCJD cases to date. These findings have important implications for the management of blood and blood products and for the handling of surgical instruments.","REFUTE (SciFact)"
+"32% of liver transplantation programs required patients to discontinue methadone treatment in 2001.","ContextChronic hepatitis C is the leading cause for liver transplantation in the United States. Intravenous drug use, the major risk factor, accounts for approximately 60% of hepatitis C virus transmission. Information from the United Network of Organ Sharing (UNOS) does not address substance use among liver transplantation patients. ObjectiveTo identify addiction-related criteria for admission to the UNOS liver transplantation waiting list and posttransplantation problems experienced by patients who are prescribed maintenance methadone. Design, Setting, and ParticipantsMail survey of all 97 adult US liver transplantation programs (belonging to UNOS) in March 2000 with telephone follow-up conducted in May and June 2000.Main Outcome MeasuresPrograms' acceptance and management of patients with past or present substance use disorder. ResultsOf the 97 programs surveyed, 87 (90%) responded. All accept applicants with a history of alcoholism or other addictions, including heroin dependence. Eighty-eight percent of the responding programs require at least 6 months of abstinence from alcohol; 83% from illicit drugs. Ninety-four percent have addiction treatment requirements. Consultations from substance abuse specialists are obtained by 86%. Patients receiving methadone maintenance are accepted by 56% of the responding programs. Approximately 180 patients receiving methadone maintenance are reported to have undergone liver transplantation. ConclusionsMost liver transplantation programs have established policies for patients with substance use disorders. Opiate-dependent patients receiving opiate replacement therapy seem underrepresented in transplantation programs. Little anecdotal evidence for negative impact of opiate replacement therapy on liver transplantation outcome was found. Policies requiring discontinuation of methadone in 32% of all programs contradict the evidence base for efficacy of long-term replacement therapies and potentially result in relapse of previously stable patients.","SUPPORT (SciFact)"
+"Several studies have also shown the association of non-coding RNAs in colorectal carcinogenesis through the stimulation or inhibition of apoptosis, cell proliferation, differentiation, invasion and metastasis","Accumulating evidence indicates that lncRNAs could play a critical role in regulation of cellular processes such as cell growth and apoptosis as well as cancer progression and metastasis. In colon cancer, a recent report indicated that miR-211 promotes cell proliferation, tumor growth and cell migration of HCT-116 cells. Although they are less well characterized compared with small non- coding microRNAs (1–5), increasing evidence suggests that lncRNAs could play a critical role in regulation of diverse cellular processes such as stem cell pluripotency, development, cell growth and apoptosis and cancer metastasis (6–13). For example, miR-211 enhances the proliferation, migration and anchorage-independent colony formation of oral carcinoma cells (35). Alterations in the primary structure, secondary structure and expression levels of lncRNAs as well as their cognate RNA-binding proteins are often associated with human diseases, in particular cancer (36).","SUPPORT (CitInt)"
+"This high AMP/ATP ratio activates the phosphorylation of AMPK, a master energy sensor within cell, and then pAMPK inhibits mTOR signaling by activating TSC2 and subsequently inhibiting Rheb","When electron transport function is inhibited, the ATP synthase can function in reverse such that it uses ATP generated by glycolysis to pump protons across the inner mitochondrial membrane, maintaining membrane potential (Appleby et al., 1999). This latter hypothesis has been questioned as cancer cells have the ability to survive on ATP produced exclusively by glycolysis. The ATP synthase inhibitor, Oligomycin A, diminished TMRE fluorescence in Control-HCT 116 p53−/− cells treated with metformin suggesting that in the presence of metformin, intact cells maintain their mitochondrial membrane potential by reversal of the ATP synthase (Figure 4E). Metformin inhibits cellular proliferation and pro- proliferative signaling via complex I inhibition. It is likely that metformin acts upstream of this site, inhibiting complex I activity while also inhibiting ROS generation.","NEI (CitInt)"

examples/inaccurate/log.csv

@@ -0,0 +1,2 @@
+claim,evidence,label
+"In this way, loc285194 functions as a downstream p53 effector that exerts its anti-proliferative role by binding miR-211 in CRC and miR-23b in GC","We present evidence that loc285194 is a direct target for p53 and functions as a tumor suppressor in part through negative regulation of miR-211. Together, these results suggest that loc285194 is a p53-regulated tumor suppressor, which acts in part through repression of miR-211. Therefore, loc285194 suppresses tumor cell growth not only in vitro but also in vivo, further suggesting that loc285194 is a p53 downstream effector, functioning as a tumor suppressor. Finally, we demonstrate that loc285194 negatively regulates miR-211, which may in part account for loc285194-mediated cell growth inhibition. Together, these results suggest that both loc285194 and miR-211 are associated with the RISC complex through which loc285194 is able to reduce the miR-211 level and vice versa.","REFUTE (CitInt)"

examples/retrieval/log.csv

@@ -0,0 +1,2 @@
+pdf_file,claim,top_k,label
+https://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0030197&type=printable,"Falsified artemisinin family drugs with no active ingredient can be life-threatening.",5,SciFact

requirements.txt

@@ -0,0 +1,6 @@
+pandas
+gradio
+pymupdf
+unidecode
+nltk
+bm25s

retrieval.py

@@ -0,0 +1,51 @@
+import re
+import fitz  # pip install pymupdf
+from unidecode import unidecode
+from nltk.tokenize import sent_tokenize
+import bm25s
+
+
+def retrieve_from_pdf(pdf_file, query, k=10):
+
+    # Get PDF file as binary
+    with open(pdf_file, mode="rb") as f:
+        pdf_file_bytes = f.read()
+
+    # Extract text from the PDF
+    pdf_doc = fitz.open(stream=pdf_file_bytes, filetype="pdf")
+    pdf_text = ""
+    for page_num in range(pdf_doc.page_count):
+        page = pdf_doc.load_page(page_num)
+        pdf_text += page.get_text("text")
+
+    # Clean text
+    # pdf_text = 'In §3.1, we find\nthat dis-\ntractor abstracts.')
+    # clean_text = 'In SS3.1, we find that distractor abstracts.'
+    # Remove hyphens at end of lines
+    clean_text = re.sub("-\n", "", pdf_text)
+    # Replace remaining newline characters with space
+    clean_text = re.sub("\n", " ", clean_text)
+    # Replace unicode with ascii
+    clean_text = unidecode(clean_text)
+
+    # Parse text into sentences to build the corpus
+    corpus = sent_tokenize(clean_text)
+    # Tokenize the corpus
+    corpus_tokens = bm25s.tokenize(corpus, stopwords="en")
+    # Initialize the BM25 model
+    retriever = bm25s.BM25()
+    retriever.index(corpus_tokens, show_progress=False)
+    # Tokenize the query
+    query_tokens = bm25s.tokenize(query)
+
+    # Get top-k results
+    # Use int(k) in case we get str value (as in retrieval example)
+    results, scores = retriever.retrieve(query_tokens, corpus=corpus, k=int(k))
+    ## Print results
+    # for i in range(results.shape[1]):
+    #    doc, score = results[0, i], scores[0, i]
+    #    print(f"Rank {i+1} (score: {score:.2f}): {doc}")
+
+    # Join sentences and return results
+    results = " ".join(results[0])
+    return results