Update pages/Comparision.py

pages/Comparision.py

@@ -1,138 +1,105 @@
 import streamlit as st
 import requests
-from transformers import pipeline
-import concurrent.futures
 import os
-import json
 from dotenv import load_dotenv
-from requests.exceptions import JSONDecodeError
+from nltk.corpus import stopwords
+from fuzzywuzzy import fuzz
+from rake_nltk import Rake
+import nltk
+from openai import OpenAI
 
 # Load environment variables
 load_dotenv()
 
-# Initialize Hugging Face API for Llama 3
-HF_API_URL = "https://api-inference.huggingface.co/v1"
-HF_API_KEY = os.getenv('HFSecret')
+# Download NLTK resources
+nltk.download('punkt')
+nltk.download('stopwords')
 
-# Initialize pipelines for Transformers
-pipe_sent_transformers = pipeline('sentiment-analysis')
-pipe_summ_transformers = pipeline("summarization", model="facebook/bart-large-cnn")
+# Initialize OpenAI client for Hugging Face Llama 3
+client = OpenAI(
+    base_url="https://api-inference.huggingface.co/v1",
+    api_key=os.environ.get('HFSecret')  # Replace with your token
+)
 
-# Define the Llama 3 model ID
-LLAMA_MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
+# Define the Llama 3 model repo ID
+repo_id = "meta-llama/Meta-Llama-3-8B-Instruct"
 
-# Function to fetch text content from Transformers app
+# Function to fetch text content based on selected option
 def fetch_text_content(selected_option):
-    options_urls = {
-        'Appreciation Letter': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Appreciation_Letter.txt",
+    url_mapping = {
+        'Apprecitation Letter': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Appreciation_Letter.txt",
         'Regret Letter': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Regret_Letter.txt",
         'Kindness Tale': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Kindness_Tale.txt",
         'Lost Melody Tale': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Lost_Melody_Tale.txt",
         'Twitter Example 1': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Twitter_Example_1.txt",
         'Twitter Example 2': "https://raw.githubusercontent.com/peteciank/public_files/main/Transformers/Twitter_Example_2.txt"
     }
-    return requests.get(options_urls[selected_option]).text if selected_option in options_urls else ""
-
-# Function to analyze sentiment using Llama
-def analyze_with_llama(text):
-    headers = {"Authorization": f"Bearer {HF_API_KEY}"}
-    data = {
-        "inputs": text,
-        "options": {
-            "use_cache": False,
-            "wait_for_model": True
-        }
+    return requests.get(url_mapping.get(selected_option, "")).text
+
+# Function to extract keywords
+def extract_keywords(text):
+    r = Rake()
+    r.extract_keywords_from_text(text)
+    phrases_with_scores = r.get_ranked_phrases_with_scores()
+    stop_words = set(stopwords.words('english'))
+    keywords = [(score, phrase) for score, phrase in phrases_with_scores if phrase.lower() not in stop_words]
+    keywords.sort(key=lambda x: x[0], reverse=True)
+
+    unique_keywords = []
+    seen_phrases = set()
+    for score, phrase in keywords:
+        if phrase not in seen_phrases:
+            similar_phrases = [seen_phrase for seen_phrase in seen_phrases if fuzz.ratio(phrase, seen_phrase) > 70]
+            merged_phrase = max([phrase] + similar_phrases, key=len) if similar_phrases else phrase
+            unique_keywords.append((score, merged_phrase))
+            seen_phrases.add(phrase)
+    return unique_keywords[:10]
+
+# Function to interact with Llama 3 for analysis
+def llama3_analysis(text, task):
+    prompt_mapping = {
+        "sentiment": f"Analyze the sentiment of the following text: {text}",
+        "summarization": f"Summarize the following text: {text}"
     }
-    
+
+    prompt = prompt_mapping[task]
+
     try:
-        response = requests.post(f"{HF_API_URL}/models/{LLAMA_MODEL_ID}", headers=headers, json=data)
-        response.raise_for_status()
-        return response.json()  # Ensure valid JSON
-    except (requests.RequestException, json.JSONDecodeError):
-        return {"error": "Error occurred while processing Llama model response."}
-
-# Function to run Transformer-based analysis
-def transformer_analysis(text):
-    # Sentiment analysis
-    sentiment_result = pipe_sent_transformers(text)
-    sentiment_score = sentiment_result[0]['score']
-    sentiment_label = sentiment_result[0]['label']
-    
-    # Summarization
-    summary_result = pipe_summ_transformers(text)
-    summary = summary_result[0]['summary_text']
-    
-    return sentiment_score, sentiment_label, summary
-
-# Function to run Llama-based analysis
-def llama_analysis(text):
-    llama_response = analyze_with_llama(text)
-    
-    if "error" in llama_response:
-        return "Error", "Error", "Error"
-    
-    # Extract sentiment and summary if valid JSON
-    sentiment_label = llama_response.get('sentiment', 'UNKNOWN')
-    sentiment_score = llama_response.get('sentiment_score', 0.0)
-    summary = llama_response.get('summary', 'No summary available.')
-    
-    return sentiment_score, sentiment_label, summary
-
-# Streamlit app layout with two columns
-st.title("Parallel Sentiment Analysis with Transformers and Llama")
-
-# Select text to analyze from dropdown
-options = ['None', 'Appreciation Letter', 'Regret Letter', 'Kindness Tale', 'Lost Melody Tale', 'Twitter Example 1', 'Twitter Example 2']
+        response = client.completions.create(
+            model=repo_id,
+            prompt=prompt,
+            max_tokens=500,
+            temperature=0.5
+        )
+        return response.choices[0].text.strip()
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+# Streamlit App UI
+st.title("Sentiment Analysis & Summarization with Llama 3")
+
+# Dropdown menu to select the text source
+options = ['None', 'Apprecitation Letter', 'Regret Letter', 'Kindness Tale', 'Lost Melody Tale', 'Twitter Example 1', 'Twitter Example 2']
 selected_option = st.selectbox("Select a preset option", options)
 
-# Fetch text content for analysis
+# Fetch the text based on selection
 jd = fetch_text_content(selected_option)
+
+# Text area for manual input or displaying fetched content
 text = st.text_area('Enter the text to analyze', jd)
 
 if st.button("Start Analysis"):
-    # Set up the two columns for parallel analysis
-    col1, col2 = st.columns(2)
-
-    with st.spinner("Running sentiment analysis..."):
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            # Execute analyses in parallel
-            future_transformer = executor.submit(transformer_analysis, text)
-            future_llama = executor.submit(llama_analysis, text)
-
-            # Retrieve results from both transformers and Llama
-            sentiment_score_transformer, sentiment_label_transformer, summary_transformer = future_transformer.result()
-            sentiment_score_llama, sentiment_label_llama, summary_llama = future_llama.result()
-
-
-
-
-    # Ensure that the score is properly handled as a float, or display the string as-is
-    def display_score(score):
-        try:
-            # Attempt to format as float if it's a valid number
-            return f"{float(score):.2f}"
-        except ValueError:
-            # If it's not a number, just return the score as is (probably a string error message)
-            return score
-    
-    # Display results for Transformers-based analysis in the first column
-    with col1:
-        st.subheader("Transformers Analysis")
-        with st.expander("Sentiment Analysis - Transformers"):
-            sentiment_emoji = '😊' if sentiment_label_transformer == 'POSITIVE' else '😞'
-            st.write(f"Sentiment: {sentiment_label_transformer} ({sentiment_emoji})")
-            st.write(f"Score: {display_score(sentiment_score_transformer)}")  # Use the display_score function
-    
-        with st.expander("Summarization - Transformers"):
-            st.write(summary_transformer)
-    
-    # Display results for Llama-based analysis in the second column
-    with col2:
-        st.subheader("Llama Analysis")
-        with st.expander("Sentiment Analysis - Llama"):
-            sentiment_emoji = '😊' if sentiment_label_llama == 'POSITIVE' else '😞'
-            st.write(f"Sentiment: {sentiment_label_llama} ({sentiment_emoji})")
-            st.write(f"Score: {display_score(sentiment_score_llama)}")  # Use the display_score function
-    
-        with st.expander("Summarization - Llama"):
-            st.write(summary_llama)
+    with st.spinner("Analyzing Sentiment..."):
+        sentiment_result = llama3_analysis(text, "sentiment")
+        with st.expander("Sentiment Analysis - ✅ Completed", expanded=False):
+            st.write(sentiment_result)
+
+    with st.spinner("Summarizing..."):
+        summary_result = llama3_analysis(text, "summarization")
+        with st.expander("Summarization - ✅ Completed", expanded=False):
+            st.write(summary_result)
+
+    with st.spinner("Extracting Keywords..."):
+        keywords = extract_keywords(text)
+        with st.expander("Keywords Extraction - ✅ Completed", expanded=False):
+            st.write([kw[1] for kw in keywords])