Adding Keyword extract options, additional elements show checkboxes

app.py

@@ -17,8 +17,16 @@ from nltk.corpus import wordnet
 import random
 from sense2vec import Sense2Vec
 import sense2vec
+from wordcloud import WordCloud
+import matplotlib.pyplot as plt
+print("***************************************************************")
+
+st.set_page_config(
+    page_title="Question Generator",
+    initial_sidebar_state="collapsed",
+)
 # Load spaCy model
-nlp = spacy.load("en_core_web_sm")
+nlp = spacy.load("en_core_web_md")
 # s2v = Sense2Vec.from_disk(self=Sense2Vec,path='s2v_old')
 
 s2v = sense2vec.Sense2Vec().from_disk('s2v_old')
@@ -34,30 +42,35 @@ def load_model():
     return model, tokenizer
 
 # Function to extract keywords using combined techniques
-def extract_keywords(text):
+def extract_keywords(text, extract_all):
+    doc = nlp(text)
+    spacy_keywords = set([ent.text for ent in doc.ents])
+    spacy_entities = spacy_keywords
+    print(f"\n\nSpacy Entities: {spacy_entities} \n\n")  
+
+    # Use Only Spacy Entities
+    if extract_all is False:
+        return list(spacy_entities) 
+    
     # Use RAKE
     rake = Rake()
     rake.extract_keywords_from_text(text)
     rake_keywords = set(rake.get_ranked_phrases())
-    
+    print(f"\n\nRake Keywords: {rake_keywords} \n\n")
     # Use spaCy for NER and POS tagging
-    doc = nlp(text)
-    spacy_keywords = set([ent.text for ent in doc.ents])
     spacy_keywords.update([token.text for token in doc if token.pos_ in ["NOUN", "PROPN", "VERB", "ADJ"]])
-    
+    print(f"\n\nSpacy Keywords: {spacy_keywords} \n\n")
     # Use TF-IDF
     vectorizer = TfidfVectorizer(stop_words='english')
     X = vectorizer.fit_transform([text])
     tfidf_keywords = set(vectorizer.get_feature_names_out())
-    
+    print(f"\n\nTFIDF Entities: {tfidf_keywords} \n\n")
+
     # Combine all keywords
     combined_keywords = rake_keywords.union(spacy_keywords).union(tfidf_keywords)
     
     return list(combined_keywords)
 
-# Load spaCy model (medium-sized model with word vectors)
-nlp = spacy.load("en_core_web_md")
-
 def get_similar_words_sense2vec(word, n=3):
     # Try to find the word with its most likely part-of-speech
     word_with_pos = word + "|NOUN"
@@ -140,7 +153,6 @@ def entity_linking(keyword):
     return None
 
 # Function to generate questions using beam search
-@st.cache_data
 def generate_question(context, answer, num_beams):
     input_text = f"<context> {context} <answer> {answer}"
     input_ids = tokenizer.encode(input_text, return_tensors='pt')
@@ -169,6 +181,19 @@ def export_to_pdf(data):
     # pdf.output("questions.pdf")
     return pdf.output(name='questions.pdf',dest='S').encode('latin1')
 
+def display_word_cloud(generated_questions):
+    word_frequency = {}
+    for question in generated_questions:
+        words = question.split()
+        for word in words:
+            word_frequency[word] = word_frequency.get(word, 0) + 1
+
+    wordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(word_frequency)
+    plt.figure(figsize=(10, 5))
+    plt.imshow(wordcloud, interpolation='bilinear')
+    plt.axis('off')
+    st.pyplot()
+
 if 'data' not in st.session_state:
     st.session_state.data = None
 
@@ -182,14 +207,21 @@ with st.sidebar:
     num_beams = st.slider("Select number of beams for question generation", min_value=1, max_value=10, value=5)
     context_window_size = st.slider("Select context window size (number of sentences before and after)", min_value=1, max_value=5, value=1)
     num_questions = st.slider("Select number of questions to generate", min_value=1, max_value=1000, value=5)
-    question_complexity = st.selectbox("Select question complexity", ["Simple", "Intermediate", "Complex"])
+    with st.expander("Choose the Additional Elements to show"):
+        show_context = st.checkbox("Context",True)
+        show_answer = st.checkbox("Answer",True)
+        show_options = st.checkbox("Options",False)
+        show_entity_link = st.checkbox("Enitity Link For Wikipedia",True)
+    extract_all_keywords = st.toggle("Extract max Keywords",value=False)
+
 if st.button("Generate Questions"):
     if text:
         model, tokenizer = load_model()
-        keywords = extract_keywords(text)
+        keywords = extract_keywords(text,extract_all_keywords)
+        print(f"\n\nFinal Keywords in Main Function: {keywords}\n\n")
         keyword_sentence_mapping = map_keywords_to_sentences(text, keywords, context_window_size)
         
-        st.subheader("Generated Questions:")
+        st.subheader("Generated Questions:",divider='blue')
         data = []
         for i, (keyword, context) in enumerate(keyword_sentence_mapping.items()):
             if i >= num_questions:
@@ -197,22 +229,26 @@ if st.button("Generate Questions"):
             linked_entity = entity_linking(keyword)
             question = generate_question(context, keyword, num_beams=num_beams)
             options = generate_options(keyword, context)
+            st.subheader(body=f":orange[Q{i+1}:] {question}")
 
-            st.write(f"**Context:** {context}")
-            st.write(f"**Answer:** {keyword}")
-            st.write(f"**Question:** {question}")
-            st.write(f"**Options:**")
-            for j, option in enumerate(options):
-                st.write(f"{chr(65+j)}. {option}")
-
-            if linked_entity:
-                st.write(f"**Entity Link:** {linked_entity}")
+            if show_context is True:
+                st.write(f"**Context:** {context}")
+            if show_answer is True:
+                st.write(f"**Answer:** {keyword}")
+            if show_options is True:
+                st.write(f"**Options:**")
+                for j, option in enumerate(options):
+                    st.write(f"{chr(65+j)}. {option}")
+            if show_entity_link is True:
+                if linked_entity:
+                    st.write(f"**Entity Link:** {linked_entity}")
             st.write("---")
             data.append((context, keyword, question, options))
 
         # Add the data to session state
         st.session_state.data = data
-        
+        # display_word_cloud()
+        print(data)
         # Export buttons
         if st.session_state.data is not None:
             with st.sidebar:
@@ -227,4 +263,5 @@ if st.button("Generate Questions"):
 
     
     else:
-        st.write("Please enter some text to generate questions.")
+        st.write("Please enter some text to generate questions.")
+    print("********************************************************************************")