add app and requirements

.gitignore

@@ -0,0 +1 @@
+.venv

app.py

@@ -0,0 +1,186 @@
+import streamlit as st
+import os
+import zipfile
+from io import BytesIO
+from PyPDF2 import PdfReader
+from keybert import KeyBERT
+from sentence_transformers import SentenceTransformer, util
+
+# Initialize KeyBERT and Sentence Transformer
+kw_model = KeyBERT()
+semantic_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+
+def main():
+    st.title("PDF Topic Grouping App")
+
+    # Step 1: Upload PDFs
+    uploaded_files = st.file_uploader("Upload PDFs", type="pdf", accept_multiple_files=True)
+
+    if not uploaded_files:
+        st.info("Please upload PDFs to continue.")
+        return
+
+    # Extract text and keywords from PDFs
+    pdf_texts = {}
+    keywords_set = set()
+
+    # Initialize a progress bar
+    progress = st.progress(0)
+    total_files = len(uploaded_files)
+    processed_files = 0
+
+    st.info("Extracting keywords from PDFs...")
+    for uploaded_file in uploaded_files:
+        pdf_name = uploaded_file.name
+        try:
+            # Read PDF content
+            reader = PdfReader(uploaded_file)
+            text = ""
+            for page in reader.pages:
+                text += page.extract_text()
+            pdf_texts[pdf_name] = text.lower()
+
+            # Extract keywords using KeyBERT
+            extracted_keywords = kw_model.extract_keywords(text, top_n=5)
+            for kw, _ in extracted_keywords:
+                keywords_set.add(kw.lower())
+
+        except Exception as e:
+            st.error(f"Failed to process {pdf_name}: {e}")
+            continue
+
+        # Update progress
+        processed_files += 1
+        progress.progress(processed_files / total_files)
+
+    # Mark progress as complete
+    progress.progress(1.0)
+
+    # Display extracted keywords and let the user select topics
+    st.write("Extracted Keywords:")
+    selected_keywords = st.multiselect(
+        "Select at least two keywords/topics for grouping:",
+        list(keywords_set),
+        default=list(keywords_set)[:2]
+    )
+
+    # Add a confirmation button to proceed
+    if st.button("Confirm Keyword Selection"):
+        if len(selected_keywords) < 2:
+            st.error("Please select at least two keywords.")
+        else:
+            st.success("Keyword selection confirmed. Processing PDFs...")
+            proceed_with_keywords = True
+    else:
+        proceed_with_keywords = False
+
+    # Ensure the process does not continue until confirmation
+    if not proceed_with_keywords:
+        st.stop()
+
+    # Initialize a progress bar for embedding precomputation
+    st.info("Precomputing embeddings for all PDFs...")
+    progress = st.progress(0)
+    total_pdfs = len(pdf_texts)
+    processed_pdfs = 0
+
+    pdf_embeddings = {}
+    for pdf_name, text in pdf_texts.items():
+        try:
+            # Compute embedding for the PDF
+            pdf_embeddings[pdf_name] = semantic_model.encode(text, convert_to_tensor=True)
+        except Exception as e:
+            st.error(f"Failed to compute embedding for {pdf_name}: {e}")
+            continue
+
+        # Update progress
+        processed_pdfs += 1
+        progress.progress(processed_pdfs / total_pdfs)
+
+    # Mark progress as complete
+    progress.progress(1.0)
+
+    # Initialize a progress bar for keyword embedding precomputation
+    st.info("Precomputing embeddings for selected keywords...")
+    progress = st.progress(0)
+    total_keywords = len(selected_keywords)
+    processed_keywords = 0
+
+    keyword_embeddings = {}
+    for keyword in selected_keywords:
+        try:
+            # Compute embedding for the keyword
+            keyword_embeddings[keyword] = semantic_model.encode(keyword, convert_to_tensor=True)
+        except Exception as e:
+            st.error(f"Failed to compute embedding for keyword '{keyword}': {e}")
+            continue
+
+        # Update progress
+        processed_keywords += 1
+        progress.progress(processed_keywords / total_keywords)
+
+    # Mark progress as complete
+    progress.progress(1.0)
+
+    # Group PDFs by the most relevant topic
+    pdf_groups = {keyword: [] for keyword in selected_keywords}
+    st.info("Assigning PDFs to the most relevant topic...")
+
+    for pdf_name, text_embedding in pdf_embeddings.items():
+        max_similarity = -1
+        best_keyword = None
+
+        # Find the most similar keyword for this PDF
+        for keyword, keyword_embedding in keyword_embeddings.items():
+            similarity = util.pytorch_cos_sim(text_embedding, keyword_embedding).item()
+            if similarity > max_similarity:
+                max_similarity = similarity
+                best_keyword = keyword
+
+        # Assign the PDF to the best matching keyword
+        if best_keyword:
+            pdf_groups[best_keyword].append(pdf_name)
+
+    # Save grouped PDFs into folders
+    output_folder = "grouped_pdfs"
+    os.makedirs(output_folder, exist_ok=True)
+    for keyword, pdf_names in pdf_groups.items():
+        keyword_folder = os.path.join(output_folder, keyword)
+        os.makedirs(keyword_folder, exist_ok=True)
+        for pdf_name in pdf_names:
+            try:
+                matched_file = next(f for f in uploaded_files if f.name == pdf_name)
+                with open(os.path.join(keyword_folder, pdf_name), "wb") as f:
+                    f.write(matched_file.getvalue())
+            except StopIteration:
+                st.error(f"File {pdf_name} not found in uploaded files.")
+                continue
+
+    # Zip the folders
+    zip_buffer = BytesIO()
+    with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+        for root, _, files in os.walk(output_folder):
+            for file in files:
+                file_path = os.path.join(root, file)
+                zip_file.write(file_path, os.path.relpath(file_path, output_folder))
+    zip_buffer.seek(0)
+
+    # Clean up temporary folders
+    for root, dirs, files in os.walk(output_folder, topdown=False):
+        for file in files:
+            os.remove(os.path.join(root, file))
+        for dir in dirs:
+            os.rmdir(os.path.join(root, dir))
+    os.rmdir(output_folder)
+
+    # Step 4: Download zipped file
+    st.success("PDFs processed and grouped successfully!")
+    st.download_button(
+        label="Download Grouped PDFs",
+        data=zip_buffer,
+        file_name="grouped_pdfs.zip",
+        mime="application/zip"
+    )
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,4 @@
+streamlit
+PyPDF2
+keybert
+sentence-transformers