Update app.py

app.py

@@ -2,22 +2,19 @@ import gradio as gr
 from datasets import load_dataset
 import pandas as pd
 import sys
-import os
-from datetime import datetime
 import subprocess
+from datetime import datetime
 from huggingface_hub import HfApi
 
 def get_newest_file(repo_id, prefix):
     """Get the newest file with given prefix from HuggingFace repo"""
     api = HfApi()
     files = api.list_repo_files(repo_id, repo_type="dataset")
-    # Filter files by prefix
     relevant_files = [f for f in files if f.startswith(prefix)]
     
     if not relevant_files:
         return None
     
-    # Sort files by modification time (newest first)
     file_dates = []
     for filename in relevant_files:
         try:
@@ -33,91 +30,114 @@ def get_newest_file(repo_id, prefix):
     newest_file = sorted(file_dates, key=lambda x: x[0], reverse=True)[0][1]
     return newest_file
 
-def load_and_show_data(repo_id, file_path):
-    """Load data from HuggingFace and return top 3 rows"""
+def load_data(repo_id, file_path):
+    """Load data from HuggingFace and return as DataFrame"""
     try:
         dataset = load_dataset(repo_id, data_files={'train': file_path}, split='train')
         df = pd.DataFrame(dataset)
-        return df.head(3).to_string()
+        return df.head(3)
     except Exception as e:
-        return f"Error loading data: {str(e)}"
+        return pd.DataFrame({'Error': [str(e)]})
 
 def praw_new_data():
     """Execute praw.py and show the latest data"""
     try:
         # Execute praw.py
         subprocess.run([sys.executable, "praw.py"], check=True)
-        
-        # Get newest crawled file
-        repo_id = "Vera-ZWY/reddite2024elections_submissions"
-        newest_file = get_newest_file(repo_id, "submissions/df_24")
-        
-        if newest_file:
-            return f"Successfully crawled new data!\n\nTop 3 rows of new data:\n{load_and_show_data(repo_id, newest_file)}"
-        else:
-            return "Error: No crawled data files found"
-    except subprocess.CalledProcessError as e:
-        return f"Error executing praw.py: {str(e)}"
+        success_message = "✅ Successfully crawled new data!"
     except Exception as e:
-        return f"Error: {str(e)}"
+        success_message = f"❌ Error executing praw.py: {str(e)}"
+    
+    # Load and return latest data
+    repo_id = "Vera-ZWY/reddite2024elections_submissions"
+    newest_file = get_newest_file(repo_id, "submissions/df_")
+    
+    if newest_file:
+        df = load_data(repo_id, newest_file)
+        return success_message, df, load_merged_data()[1]  # Return current merged data state
+    else:
+        return "No crawled data files found", pd.DataFrame(), load_merged_data()[1]
 
 def merge_data():
     """Execute merge.py and show the latest merged data"""
     try:
         # Execute merge.py
         subprocess.run([sys.executable, "merge.py"], check=True)
-        
-        # Get newest merged file
-        repo_id = "Vera-ZWY/reddite2024elections_submissions"
-        old_merged =  "submission/merged_reddit_data.csv"
-        
-        if newest_merged:
-            return f"Successfully merged data!\n\nTop 3 rows of merged data:\n{load_and_show_data(repo_id, newest_merged)}"
-        else:
-            return "Error: No merged data files found"
-    except subprocess.CalledProcessError as e:
-        return f"Error executing merge.py: {str(e)}"
+        success_message = "✅ Successfully merged data!"
     except Exception as e:
-        return f"Error: {str(e)}"
+        success_message = f"❌ Error executing merge.py: {str(e)}"
+    
+    # Load and return latest merged data
+    merged_df = load_merged_data()[1]
+    crawled_df = load_crawled_data()[1]
+    return success_message, crawled_df, merged_df
 
-def load_latest_data():
-    """Load and display the latest data from both sources"""
+def load_crawled_data():
+    """Load latest crawled data"""
     repo_id = "Vera-ZWY/reddite2024elections_submissions"
+    newest_file = get_newest_file(repo_id, "submissions/df_24")
     
-    # Get latest crawled data
-    newest_crawled = get_newest_file(repo_id, "submissions/df_24")
-    crawled_data = "No crawled data available"
-    if newest_crawled:
-        crawled_data = f"Latest crawled data ({newest_crawled}):\n{load_and_show_data(repo_id, newest_crawled)}"
+    if newest_file:
+        return f"Latest crawled data ({newest_file}):", load_data(repo_id, newest_file)
+    return "No crawled data available", pd.DataFrame()
+
+def load_merged_data():
+    """Load latest merged data"""
+    repo_id = "Vera-ZWY/reddite2024elections_submissions"
+    newest_merged =  "submission/merged_reddit_data.csv"
     
-    # Get latest merged data
-    newest_merged = get_newest_file(repo_id, "merged_reddit_data")
-    merged_data = "No merged data available"
     if newest_merged:
-        merged_data = f"Latest merged data ({newest_merged}):\n{load_and_show_data(repo_id, newest_merged)}"
-    
-    return f"{crawled_data}\n\n{merged_data}"
+        return f"Latest merged data ({newest_merged}):", load_data(repo_id, newest_merged)
+    return "No merged data available", pd.DataFrame()
 
 # Create Gradio interface
 with gr.Blocks(title="Reddit Data Processing") as iface:
     gr.Markdown("# Reddit Data Processing Interface")
     
+    # Status message for operations
+    status_text = gr.Textbox(label="Status", interactive=False)
+    
+    with gr.Row():
+        with gr.Column():
+            praw_button = gr.Button("Crawl New Data", variant="primary")
+        with gr.Column():
+            merge_button = gr.Button("Merge Data", variant="primary")
+    
     with gr.Row():
         with gr.Column():
-            praw_button = gr.Button("Crawl New Data")
-            merge_button = gr.Button("Merge Data")
+            gr.Markdown("### Latest Crawled Data (Top 3 Rows)")
+            crawled_table = gr.Dataframe(
+                headers=["title", "score", "id", "url", "comms_num", "created", "body", "subreddit"],
+                value=load_crawled_data()[1],
+                wrap=True
+            )
     
-    output_text = gr.Textbox(label="Results", value=load_latest_data(), lines=20)
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("### Latest Merged Data (Top 3 Rows)")
+            merged_table = gr.Dataframe(
+                headers=["title", "score", "id", "url", "num_comments", "created", "body", "content", "subreddit"],
+                value=load_merged_data()[1],
+                wrap=True
+            )
     
     # Button click handlers
-    praw_button.click(fn=praw_new_data, outputs=output_text)
-    merge_button.click(fn=merge_data, outputs=output_text)
+    praw_button.click(
+        fn=praw_new_data,
+        outputs=[status_text, crawled_table, merged_table]
+    )
+    
+    merge_button.click(
+        fn=merge_data,
+        outputs=[status_text, crawled_table, merged_table]
+    )
     
     gr.Markdown("""
+    ## The full dataset storage at https://huggingface.co/datasets/Vera-ZWY/reddite2024elections_submissions/
     ### Instructions:
     1. Click 'Crawl New Data' to fetch new Reddit data
     2. Click 'Merge Data' to merge the latest datasets
-    3. The results will show the top 3 rows of both the latest crawled and merged data
+    3. Tables will automatically update to show the latest data
     """)
 
 # Launch the interface