Update app.py

app.py

@@ -15,15 +15,18 @@ from nltk.tokenize import sent_tokenize
 nltk.download('wordnet')
 from nltk.corpus import wordnet
 import random
-
+from sense2vec import Sense2Vec
+import sense2vec
 # Load spaCy model
 nlp = spacy.load("en_core_web_sm")
+# s2v = Sense2Vec.from_disk(self=Sense2Vec,path='s2v_old')
 
+s2v = sense2vec.Sense2Vec().from_disk('s2v_old')
 # Initialize Wikipedia API with a user agent
 user_agent = 'QGen/1.0 ([email protected])'
 wiki_wiki = wikipediaapi.Wikipedia(user_agent= user_agent,language='en')
 
-@st.cache_resource(allow_output_mutation=True)
+@st.cache_resource
 def load_model():
     model_name = "DevBM/t5-large-squad"
     model = T5ForConditionalGeneration.from_pretrained(model_name)
@@ -55,20 +58,19 @@ def extract_keywords(text):
 # Load spaCy model (medium-sized model with word vectors)
 nlp = spacy.load("en_core_web_md")
 
-def get_similar_words(word, n=3):
-    # Get the vector for the word
-    word_vector = nlp(word).vector
+def get_similar_words_sense2vec(word, n=3):
+    # Try to find the word with its most likely part-of-speech
+    word_with_pos = word + "|NOUN"
+    if word_with_pos in s2v:
+        similar_words = s2v.most_similar(word_with_pos, n=n)
+        return [word.split("|")[0] for word, _ in similar_words]
     
-    # Find similar words
-    similar_words = []
-    for w in nlp.vocab:
-        if w.has_vector and w.is_lower and w.is_alpha and w.text != word:
-            similarity = nlp(w.text).similarity(nlp(word))
-            similar_words.append((w.text, similarity))
+    # If not found, try without POS
+    if word in s2v:
+        similar_words = s2v.most_similar(word, n=n)
+        return [word.split("|")[0] for word, _ in similar_words]
     
-    # Sort by similarity and return top n
-    similar_words.sort(key=lambda x: x[1], reverse=True)
-    return [word for word, _ in similar_words[:n]]
+    return []
 
 def get_synonyms(word, n=3):
     synonyms = []
@@ -83,8 +85,8 @@ def get_synonyms(word, n=3):
 def generate_options(answer, context, n=3):
     options = [answer]
     
-    # Try to get similar words based on word vectors
-    similar_words = get_similar_words(answer, n)
+    # Try to get similar words based on sense2vec
+    similar_words = get_similar_words_sense2vec(answer, n)
     options.extend(similar_words)
     
     # If we don't have enough options, try synonyms
@@ -138,6 +140,7 @@ def entity_linking(keyword):
     return None
 
 # Function to generate questions using beam search
+@st.cache_data
 def generate_question(context, answer, num_beams):
     input_text = f"<context> {context} <answer> {answer}"
     input_ids = tokenizer.encode(input_text, return_tensors='pt')
@@ -147,7 +150,7 @@ def generate_question(context, answer, num_beams):
 
 # Function to export questions to CSV
 def export_to_csv(data):
-    df = pd.DataFrame(data, columns=["Context", "Answer", "Question"])
+    df = pd.DataFrame(data, columns=["Context", "Answer", "Question", "Options"])
     csv = df.to_csv(index=False,encoding='utf-8')
     return csv
 
@@ -157,7 +160,7 @@ def export_to_pdf(data):
     pdf.add_page()
     pdf.set_font("Arial", size=12)
     
-    for context, answer, question in data:
+    for context, answer, question, options in data:
         pdf.multi_cell(0, 10, f"Context: {context}")
         pdf.multi_cell(0, 10, f"Answer: {answer}")
         pdf.multi_cell(0, 10, f"Question: {question}")
@@ -199,13 +202,13 @@ if st.button("Generate Questions"):
             st.write(f"**Answer:** {keyword}")
             st.write(f"**Question:** {question}")
             st.write(f"**Options:**")
-            for j, option in options:
+            for j, option in enumerate(options):
                 st.write(f"{chr(65+j)}. {option}")
 
             if linked_entity:
                 st.write(f"**Entity Link:** {linked_entity}")
             st.write("---")
-            data.append((context, keyword, question))
+            data.append((context, keyword, question, options))
 
         # Add the data to session state
         st.session_state.data = data
@@ -224,4 +227,4 @@ if st.button("Generate Questions"):
 
     
     else:
-        st.write("Please enter some text to generate questions.")
+        st.write("Please enter some text to generate questions.")