Duplicate from Papajams/substantiator

Co-authored-by: Papa Jams <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Substantiator
+emoji: 🐠
+colorFrom: red
+colorTo: indigo
+sdk: streamlit
+sdk_version: 1.21.0
+app_file: app.py
+pinned: false
+license: wtfpl
+duplicated_from: Papajams/substantiator
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,85 @@
+import streamlit as st
+import requests
+import torch
+from transformers import AutoTokenizer, AutoModel
+import xml.etree.ElementTree as ET
+
+# Load SciBERT pre-trained model and tokenizer
+model_name = "allenai/scibert_scivocab_uncased"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModel.from_pretrained(model_name)
+
+def calculate_similarity(claim, document):
+    if not claim or not document:
+        return 0.0
+    # Tokenize claim and document
+    inputs = tokenizer.encode_plus(claim, document, return_tensors='pt', padding=True, truncation=True)
+    
+    # Generate embeddings for claim
+    with torch.no_grad():
+        claim_embeddings = model(**inputs)['pooler_output']
+    
+    # Generate embeddings for document
+    inputs_doc = tokenizer.encode_plus(document, return_tensors='pt', padding=True, truncation=True)
+    with torch.no_grad():
+        document_embeddings = model(**inputs_doc)['pooler_output']
+    
+    # Compute cosine similarity between embeddings
+    similarity = torch.cosine_similarity(claim_embeddings, document_embeddings).item()
+    
+    return similarity
+
+def search_arxiv(query, max_results=3):
+    base_url = "http://export.arxiv.org/api/query?"
+    query = f"search_query=all:{query}&start=0&max_results={max_results}&sortBy=relevance&sortOrder=descending"
+    
+    try:
+        response = requests.get(base_url + query)
+        if response.status_code == 200:
+            data = response.content
+
+            # Parse the XML response
+            root = ET.fromstring(data)
+
+            search_results = []
+            for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
+                result = {}
+
+                # Extract information from each entry
+                result["title"] = entry.find("{http://www.w3.org/2005/Atom}title").text
+                result["abstract"] = entry.find("{http://www.w3.org/2005/Atom}summary").text
+                result["link"] = entry.find("{http://www.w3.org/2005/Atom}link[@title='pdf']").attrib["href"]
+
+                authors = []
+                for author in entry.findall("{http://www.w3.org/2005/Atom}author"):
+                    authors.append(author.find("{http://www.w3.org/2005/Atom}name").text)
+                result["authors"] = authors
+
+                search_results.append(result)
+
+            return search_results
+    except:
+        return None
+
+def search_papers(user_input):
+    # Use the desired search function, e.g., search_arxiv
+    search_results = search_arxiv(user_input)
+    return search_results
+
+st.title('The Substantiator')
+
+user_input = st.text_input('Input your claim')
+
+if st.button('Substantiate'):
+    search_results = search_papers(user_input)
+    if search_results is not None and len(search_results) > 0:
+        with st.spinner('Searching for relevant research papers...'):
+            for result in search_results[:3]:
+                st.write(f"<a href='javascript:void(0)' onclick='window.open(\"{result['link']}\", \"_blank\");return false;'>{result['title']}</a>", unsafe_allow_html=True)
+                st.write(result["abstract"])
+                st.write("Authors: ", ", ".join(result["authors"]))
+                similarity = calculate_similarity(user_input, result["abstract"])
+                st.write("Similarity Score: ", similarity)
+                st.write("-----")
+    else:
+        st.write("No results found.")

huggingface.yaml

@@ -0,0 +1,20 @@
+title: Substantiator
+description: A Streamlit app that uses SciBERT for semantic similarity.
+tags:
+  - Scientific
+  - NLP
+pipeline:
+  - name: Streamlit
+    inputs:
+      - name: user_input
+        type: text
+        description: Input your claim
+    outputs:
+      - name: substantiation
+        type: text
+        description: Substantiation of the claim
+    resources:
+      limits:
+        cpu: 1
+        memory: 2Gi
+

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+requests
+torch
+transformers
+beautifulsoup4