added 3D interactive plot

app.py

@@ -1,4 +1,4 @@
-from flask import Flask, render_template, request, jsonify
+from flask import Flask, render_template, request, jsonify, Response
 from dotenv import load_dotenv
 import requests
 from datetime import datetime
@@ -8,45 +8,56 @@ import openai
 import numpy as np
 import pickle
 from pathlib import Path
+import umap
+import plotly.express as px
+import plotly.graph_objects as go
+import pandas as pd
+from sklearn.cluster import DBSCAN
+from sklearn.preprocessing import StandardScaler
+import time
+import queue
+import threading
 
 load_dotenv()
 
 app = Flask(__name__)
 
 # Get API keys from environment variables
-PATENTSVIEW_API_KEY = os.getenv('PATENTSVIEW_API_KEY')
+SERPAPI_API_KEY = os.getenv('SERPAPI_API_KEY')
 OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
 MAX_PATENTS = 300  # Limit number of patents to process
 CACHE_FILE = 'patent_embeddings_cache.pkl'
-QUERY_CACHE_FILE = 'query_embeddings_cache.pkl'
 
-if not PATENTSVIEW_API_KEY:
-    raise ValueError("PATENTSVIEW_API_KEY environment variable is not set")
+# Global progress queue for SSE updates
+progress_queue = queue.Queue()
+
+if not SERPAPI_API_KEY:
+    raise ValueError("SERPAPI_API_KEY environment variable is not set")
 if not OPENAI_API_KEY:
     raise ValueError("OPENAI_API_KEY environment variable is not set")
 
 # Initialize OpenAI API key
 openai.api_key = OPENAI_API_KEY
 
-def load_cache(cache_file):
+def load_cache():
     """Load cached embeddings from file"""
     try:
-        if os.path.exists(cache_file):
-            with open(cache_file, 'rb') as f:
+        if os.path.exists(CACHE_FILE):
+            with open(CACHE_FILE, 'rb') as f:
                 return pickle.load(f)
     except Exception as e:
         print(f"Error loading cache: {e}")
     return {}
 
-def save_cache(cache, cache_file):
+def save_cache(cache):
     """Save embeddings cache to file"""
     try:
-        with open(cache_file, 'wb') as f:
+        with open(CACHE_FILE, 'wb') as f:
             pickle.dump(cache, f)
     except Exception as e:
         print(f"Error saving cache: {e}")
 
-def get_embedding(text, cache, cache_file):
+def get_embedding(text, cache):
     """Get embedding for text, using cache if available"""
     if not text or text.strip() == "":
         return None
@@ -62,7 +73,7 @@ def get_embedding(text, cache, cache_file):
         embedding = response['data'][0]['embedding']
         if embedding:  # Only cache if we got a valid embedding
             cache[text] = embedding
-            save_cache(cache, cache_file)  # Save cache after each new embedding
+            save_cache(cache)  # Save cache after each new embedding
         return embedding
     except Exception as e:
         print(f"Error getting embedding: {e}")
@@ -70,119 +81,97 @@ def get_embedding(text, cache, cache_file):
 
 def search_patents(keywords, page_size=100):
     """
-    Search patents using PatentsView API with semantic search capabilities
+    Search patents using SerpApi's Google Patents API with pagination and generate embeddings
     """
-    # Load separate caches for patents and queries
-    patent_cache = load_cache(CACHE_FILE)
-    query_cache = load_cache(QUERY_CACHE_FILE)
-    
-    # Get embedding for search query
-    query_embedding = get_embedding(keywords, query_cache, QUERY_CACHE_FILE)
-    if not query_embedding:
-        return []
-    
-    # PatentsView API endpoint
-    api_url = "https://search.patentsview.org/api/v1/patent/"
-    
-    # Create a broader search query using related terms
-    # We'll search in both title and abstract with more flexible matching
-    query = {
-        "q": {
-            "_or": [
-                {"_text_any": {"patent_title": keywords.split()}},
-                {"_text_any": {"patent_abstract": keywords.split()}}
-            ]
-        },
-        "f": [
-            "patent_title",
-            "patent_abstract",
-            "patent_date",
-            "patent_id",
-            "assignees"
-        ],
-        "o": {
-            "page": 1,
-            "size": MAX_PATENTS  # Get maximum allowed patents for better semantic matching
-        }
-    }
-
+    # Load existing cache
+    embedding_cache = load_cache()
+    
     all_patents = []
-    try:
-        headers = {
-            "Content-Type": "application/json",
-            "X-Api-Key": PATENTSVIEW_API_KEY
-        }
+    page = 1
+    total_processed = 0
+    
+    while len(all_patents) < MAX_PATENTS:
+        update_progress('search', f'Fetching page {page} of patents...')
         
-        response = requests.post(api_url, json=query, headers=headers)
-        response_data = response.json()
+        # SerpApi Google Patents API endpoint
+        api_url = "https://serpapi.com/search"
         
-        if response_data.get('error'):
-            print(f"API returned error: {response_data}")
-            return []
+        params = {
+            "engine": "google_patents",
+            "q": keywords,
+            "api_key": SERPAPI_API_KEY,
+            "num": page_size,
+            "start": (page - 1) * page_size
+        }
+
+        try:
+            response = requests.get(api_url, params=params)
+            response_data = response.json()
             
-        patents_data = response_data.get('patents', [])
-        
-        # Process and embed all patents
-        for patent in patents_data:
-            # Format filing date
-            date_str = patent.get('patent_date', '')
-            filing_year = 'N/A'
-            if date_str:
-                try:
-                    filing_year = datetime.strptime(date_str, '%Y-%m-%d').year
-                except ValueError:
-                    pass
-
-            # Get first assignee organization if available
-            assignee_org = 'N/A'
-            assignees = patent.get('assignees', [])
-            if assignees and len(assignees) > 0:
-                assignee_org = assignees[0].get('assignee_organization', 'N/A')
-
-            # Format patent ID for Google Patents URL
-            patent_id = patent.get('patent_id', '')
-            if patent_id and not patent_id.startswith('US'):
-                patent_id = f"US{patent_id}"
-
-            # Combine title and abstract for embedding
-            title = patent.get('patent_title', '').strip()
-            abstract = patent.get('patent_abstract', '').strip()
-            combined_text = f"{title}\n{abstract}".strip()
-
-            # Get embedding for combined text using patent cache
-            patent_embedding = get_embedding(combined_text, patent_cache, CACHE_FILE)
+            if "error" in response_data:
+                print(f"API returned error: {response_data['error']}")
+                break
+                
+            patents_data = response_data.get('organic_results', [])
             
-            if patent_embedding:
-                # Calculate cosine similarity with query
-                similarity = np.dot(query_embedding, patent_embedding) / (
-                    np.linalg.norm(query_embedding) * np.linalg.norm(patent_embedding)
-                )
+            if not patents_data:
+                print(f"No more patents found on page {page}")
+                break
+                
+            for idx, patent in enumerate(patents_data):
+                if len(all_patents) >= MAX_PATENTS:
+                    break
+                    
+                # Format filing date
+                filing_date = patent.get('filing_date', '')
+                filing_year = 'N/A'
+                if filing_date:
+                    try:
+                        filing_year = datetime.strptime(filing_date, '%Y-%m-%d').year
+                    except ValueError:
+                        pass
+
+                # Get assignee
+                assignee = patent.get('assignee', 'N/A')
+                if isinstance(assignee, list) and assignee:
+                    assignee = assignee[0]
+
+                # Format title and abstract for embedding
+                title = patent.get('title', '').strip()
+                abstract = patent.get('snippet', '').strip()
+                combined_text = f"{title}\n{abstract}".strip()
+
+                # Get embedding for combined text
+                total_processed += 1
+                if total_processed % 10 == 0:  # Update progress every 10 patents
+                    update_progress('embedding', f'Processing patent {total_processed} of {MAX_PATENTS}...')
                 
+                embedding = get_embedding(combined_text, embedding_cache)
+
                 formatted_patent = {
                     'title': title,
-                    'assignee': assignee_org,
+                    'assignee': assignee,
                     'filing_year': filing_year,
                     'abstract': abstract,
-                    'link': f"https://patents.google.com/patent/{patent_id}",
-                    'embedding': patent_embedding,
-                    'similarity': float(similarity)  # Convert to float for JSON serialization
+                    'link': patent.get('patent_link', '') or patent.get('link', ''),
+                    'embedding': embedding
                 }
                 all_patents.append(formatted_patent)
-        
-        # Sort patents by similarity score
-        all_patents.sort(key=lambda x: x['similarity'], reverse=True)
-        
-        # Take top MAX_PATENTS results
-        all_patents = all_patents[:MAX_PATENTS]
-        print(len(all_patents))
-        
-    except Exception as e:
-        print(f"Error searching patents: {e}")
-        return []
+            
+            print(f"Retrieved {len(patents_data)} patents from page {page}")
+            
+            # Check if there are more pages
+            if not response_data.get('serpapi_pagination', {}).get('next'):
+                break
+                
+            page += 1
+            
+        except Exception as e:
+            print(f"Error searching patents: {e}")
+            break
     
-    # Save final cache states
-    save_cache(patent_cache, CACHE_FILE)
-    save_cache(query_cache, QUERY_CACHE_FILE)
+    # Save final cache state
+    save_cache(embedding_cache)
     
     print(f"Total patents retrieved and embedded: {len(all_patents)}")
     return all_patents
@@ -218,10 +207,284 @@ def generate_summary(patents):
         print(f"Error generating summary: {str(e)}")
         return "Error generating summary."
 
+def analyze_clusters(df, labels, embeddings_3d):
+    """
+    Generate descriptions for patent clusters and identify opportunity zones
+    """
+    unique_labels = np.unique(labels)
+    cluster_insights = []
+    
+    # Analyze each cluster (including noise points labeled as -1)
+    for label in unique_labels:
+        cluster_mask = labels == label
+        cluster_patents = df[cluster_mask]
+        cluster_points = embeddings_3d[cluster_mask]
+        
+        if label == -1:
+            # Analyze sparse regions (potential opportunity zones)
+            if len(cluster_patents) > 0:
+                titles = "\n".join(cluster_patents['title'].tolist())
+                assignees = ", ".join(cluster_patents['assignee'].unique())
+                years = f"{cluster_patents['year'].min()} - {cluster_patents['year'].max()}"
+                
+                prompt = f"""Analyze these {len(cluster_patents)} patents that are in sparse regions of the technology landscape:
+
+Patents:
+{titles}
+
+Key assignees: {assignees}
+Years: {years}
+
+Please provide:
+1. A brief description of these isolated technologies
+2. Potential innovation opportunities in this space
+3. Why these areas might be underexplored
+Keep the response concise (max 3 sentences per point)."""
+
+                try:
+                    response = openai.ChatCompletion.create(
+                        model="gpt-3.5-turbo",
+                        messages=[
+                            {"role": "system", "content": "You are a patent and technology expert analyzing innovation opportunities."},
+                            {"role": "user", "content": prompt}
+                        ],
+                        max_tokens=300,
+                        temperature=0.7
+                    )
+                    cluster_insights.append({
+                        'type': 'opportunity_zone',
+                        'size': len(cluster_patents),
+                        'description': response['choices'][0]['message']['content']
+                    })
+                except Exception as e:
+                    print(f"Error generating opportunity zone analysis: {e}")
+        else:
+            # Analyze regular clusters
+            if len(cluster_patents) > 0:
+                titles = "\n".join(cluster_patents['title'].tolist())
+                assignees = ", ".join(cluster_patents['assignee'].unique())
+                years = f"{cluster_patents['year'].min()} - {cluster_patents['year'].max()}"
+                
+                prompt = f"""Analyze this cluster of {len(cluster_patents)} related patents:
+
+Patents:
+{titles}
+
+Key assignees: {assignees}
+Years: {years}
+
+Please provide a concise (2-3 sentences) summary of:
+1. The main technology focus of this cluster
+2. Current development status and trends"""
+
+                try:
+                    response = openai.ChatCompletion.create(
+                        model="gpt-3.5-turbo",
+                        messages=[
+                            {"role": "system", "content": "You are a patent and technology expert analyzing innovation clusters."},
+                            {"role": "user", "content": prompt}
+                        ],
+                        max_tokens=200,
+                        temperature=0.7
+                    )
+                    cluster_insights.append({
+                        'type': 'cluster',
+                        'id': int(label),
+                        'size': len(cluster_patents),
+                        'description': response['choices'][0]['message']['content']
+                    })
+                except Exception as e:
+                    print(f"Error generating cluster analysis: {e}")
+    
+    return cluster_insights
+
+def create_3d_visualization(patents):
+    """
+    Create a 3D visualization of patent embeddings using UMAP and Plotly
+    """
+    if not patents:
+        return None
+        
+    update_progress('clustering', 'Extracting embeddings...')
+    
+    # Extract embeddings and metadata
+    embeddings = []
+    metadata = []
+    for patent in patents:
+        if patent['embedding'] is not None:
+            embeddings.append(patent['embedding'])
+            abstract = patent['abstract']
+            if len(abstract) > 200:
+                abstract = abstract[:200] + "..."
+            
+            metadata.append({
+                'title': patent['title'],
+                'assignee': patent['assignee'],
+                'year': patent['filing_year'],
+                'abstract': abstract,
+                'link': patent['link']
+            })
+    
+    if not embeddings:
+        return None
+        
+    # Convert embeddings to numpy array
+    embeddings_array = np.array(embeddings)
+    
+    update_progress('clustering', 'Applying UMAP dimensionality reduction...')
+    
+    # Apply UMAP dimensionality reduction
+    reducer = umap.UMAP(n_components=3, random_state=42)
+    embedding_3d = reducer.fit_transform(embeddings_array)
+    
+    update_progress('clustering', 'Performing DBSCAN clustering...')
+    
+    # Create DataFrame for plotting
+    df = pd.DataFrame(metadata)
+    df['x'] = embedding_3d[:, 0]
+    df['y'] = embedding_3d[:, 1]
+    df['z'] = embedding_3d[:, 2]
+    
+    # Apply DBSCAN clustering
+    scaler = StandardScaler()
+    scaled_embeddings = scaler.fit_transform(embedding_3d)
+    dbscan = DBSCAN(eps=0.75, min_samples=5)
+    clusters = dbscan.fit_predict(scaled_embeddings)
+    
+    update_progress('analysis', 'Analyzing clusters and opportunities...')
+    
+    # Print clustering statistics
+    n_clusters = len(set(clusters)) - (1 if -1 in clusters else 0)
+    n_noise = list(clusters).count(-1)
+    print(f"\nClustering Statistics:")
+    print(f"Number of clusters: {n_clusters}")
+    print(f"Number of patents in sparse regions: {n_noise}")
+    print(f"Total number of patents: {len(clusters)}")
+    
+    if n_noise == 0:
+        print("\nWarning: No sparse regions detected. Consider adjusting DBSCAN parameters.")
+        dbscan = DBSCAN(eps=0.5, min_samples=7)
+        clusters = dbscan.fit_predict(scaled_embeddings)
+        n_clusters = len(set(clusters)) - (1 if -1 in clusters else 0)
+        n_noise = list(clusters).count(-1)
+        print(f"\nRetrying with stricter parameters:")
+        print(f"Number of clusters: {n_clusters}")
+        print(f"Number of patents in sparse regions: {n_noise}")
+    
+    df['cluster'] = clusters
+    
+    update_progress('analysis', 'Generating cluster insights...')
+    
+    # Generate cluster insights
+    cluster_insights = analyze_clusters(df, clusters, embedding_3d)
+    
+    update_progress('visualization', 'Creating interactive plot...')
+    
+    # Create hover text with cluster information
+    hover_text = []
+    for idx, row in df.iterrows():
+        cluster_info = ""
+        if row['cluster'] == -1:
+            cluster_info = "<br><b>Region:</b> Sparse Area (Potential Innovation Zone)"
+        else:
+            cluster_info = f"<br><b>Cluster:</b> {row['cluster']}"
+            
+        text = (
+            f"<b>{row['title']}</b><br><br>"
+            f"<b>By:</b> {row['assignee']} ({row['year']})<br>"
+            f"{cluster_info}<br><br>"
+            f"<b>Abstract:</b><br>{row['abstract']}"
+        )
+        hover_text.append(text)
+    
+    # Create Plotly figure with clusters
+    fig = go.Figure(data=[go.Scatter3d(
+        x=df['x'],
+        y=df['y'],
+        z=df['z'],
+        mode='markers',
+        marker=dict(
+            size=10,
+            color=clusters,
+            colorscale='Viridis',
+            opacity=0.8,
+            showscale=True,
+            colorbar=dict(
+                title="Clusters<br>(-1: Opportunity Zones)",
+                tickfont=dict(size=10),
+                titlefont=dict(size=10)
+            )
+        ),
+        text=hover_text,
+        hoverinfo='text',
+        hoverlabel=dict(
+            bgcolor="white",
+            font_size=12,
+            font_family="Arial",
+            align="left"
+        ),
+        customdata=df['link'].tolist()
+    )])
+    
+    # Update layout
+    fig.update_layout(
+        title="Patent Technology Landscape with Innovation Clusters",
+        scene=dict(
+            xaxis_title="UMAP 1",
+            yaxis_title="UMAP 2",
+            zaxis_title="UMAP 3",
+            camera=dict(
+                up=dict(x=0, y=0, z=1),
+                center=dict(x=0, y=0, z=0),
+                eye=dict(x=1.5, y=1.5, z=1.5)
+            )
+        ),
+        margin=dict(l=0, r=0, b=0, t=30),
+        showlegend=False,
+        template="plotly_dark",
+        hoverlabel_align='left',
+        hoverdistance=100,
+        hovermode='closest'
+    )
+    
+    # Add hover template configuration
+    fig.update_traces(
+        hovertemplate='%{text}<extra></extra>'
+    )
+    
+    update_progress('visualization', 'Finalizing visualization...')
+    
+    return {
+        'plot': fig.to_json(),
+        'insights': cluster_insights
+    }
+
 @app.route('/')
 def home():
     return render_template('index.html')
 
+@app.route('/progress')
+def get_progress():
+    """Server-sent events endpoint for progress updates"""
+    def generate():
+        while True:
+            try:
+                data = progress_queue.get(timeout=30)  # 30 second timeout
+                if data == 'DONE':
+                    break
+                yield f"data: {json.dumps(data)}\n\n"
+            except queue.Empty:
+                break
+    return Response(generate(), mimetype='text/event-stream')
+
+def update_progress(step, status='processing'):
+    """Update progress through the progress queue"""
+    progress_queue.put({
+        'step': step,
+        'status': status,
+        'timestamp': datetime.now().strftime('%H:%M:%S')
+    })
+
 @app.route('/search', methods=['POST'])
 def search():
     keywords = request.form.get('keywords', '')
@@ -229,16 +492,45 @@ def search():
         return jsonify({'error': 'Please enter search keywords'})
     
     print(f"\nProcessing search request for keywords: {keywords}")
-    patents = search_patents(keywords)
-    if not patents:
-        return jsonify({'error': 'No patents found or an error occurred'})
     
-    # Generate summary using ChatGPT
-    # summary = generate_summary(patents)
-    return jsonify({
-        'patents': patents,
-        'summary': None  # Set to None since we're not generating summaries currently
-    })
+    try:
+        # Clear any existing progress updates
+        while not progress_queue.empty():
+            progress_queue.get_nowait()
+            
+        # Search for patents
+        update_progress('search')
+        patents = search_patents(keywords)
+        if not patents:
+            return jsonify({'error': 'No patents found or an error occurred'})
+        
+        # Generate embeddings
+        update_progress('embedding')
+        
+        # Cluster analysis
+        update_progress('clustering')
+        
+        # Innovation analysis
+        update_progress('analysis')
+        
+        # Create visualization
+        update_progress('visualization')
+        viz_data = create_3d_visualization(patents)
+        if not viz_data:
+            return jsonify({'error': 'Error creating visualization'})
+        
+        # Signal completion
+        progress_queue.put('DONE')
+        
+        return jsonify({
+            'visualization': viz_data['plot'],
+            'insights': viz_data['insights']
+        })
+        
+    except Exception as e:
+        print(f"Error processing request: {e}")
+        progress_queue.put('DONE')
+        return jsonify({'error': str(e)})
 
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=7860) 

requirements.txt

@@ -1,10 +1,9 @@
 flask==2.0.1
-Werkzeug==2.0.3
-requests==2.31.0
-gunicorn==20.1.0
-itsdangerous==2.0.1
-Jinja2==3.0.1
-MarkupSafe==2.0.1
+python-dotenv==0.19.0
+requests==2.26.0
 openai==0.28.1
-python-dotenv==1.0.1
-numpy==1.26.1
+numpy==1.24.3
+pandas==2.0.3
+umap-learn==0.5.3
+plotly==5.3.1
+scikit-learn==1.3.0

templates/index.html

@@ -3,43 +3,272 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Patent Explorer</title>
+    <title>Patent Technology Landscape</title>
     <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/tailwind.min.css" rel="stylesheet">
     <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+    <style>
+        .visualization-container {
+            width: 100%;
+            height: 50vh;
+            margin-bottom: 20px;
+            background-color: #1a1a1a;
+            border-radius: 8px;
+            overflow: hidden;
+        }
+        .insights-panel {
+            background-color: #2d2d2d;
+            border-radius: 8px;
+            height: calc(120vh - 40px);
+            overflow-y: auto;
+            transition: all 0.3s ease;
+        }
+        .cluster-card {
+            background-color: #3d3d3d;
+            border-radius: 6px;
+            margin-bottom: 10px;
+            transition: all 0.2s ease;
+        }
+        .cluster-card:hover {
+            transform: translateY(-2px);
+            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        }
+        .opportunity-card {
+            background-color: #2d4a3e;
+            border-radius: 6px;
+            margin-bottom: 10px;
+            transition: all 0.2s ease;
+        }
+        .opportunity-card:hover {
+            transform: translateY(-2px);
+            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        }
+        .loading-container {
+            background-color: #2d2d2d;
+            border-radius: 8px;
+            padding: 1.5rem;
+            margin-bottom: 1rem;
+        }
+        .progress-bar {
+            width: 100%;
+            height: 8px;
+            background-color: #4a5568;
+            border-radius: 4px;
+            margin-bottom: 1rem;
+            overflow: hidden;
+        }
+        .progress-fill {
+            height: 100%;
+            width: 0;
+            background-color: #4299e1;
+            transition: width 0.3s ease;
+        }
+        .status-list {
+            max-height: 150px;
+            overflow-y: auto;
+        }
+        .status-item {
+            display: flex;
+            align-items: center;
+            margin-bottom: 0.75rem;
+            padding: 0.5rem;
+            border-radius: 4px;
+            background-color: #374151;
+            opacity: 0.6;
+            transition: all 0.3s ease;
+        }
+        .status-item.active {
+            opacity: 1;
+            background-color: #3b4f7d;
+        }
+        .status-item.complete {
+            opacity: 0.8;
+            background-color: #2d4a3e;
+        }
+        .status-icon {
+            width: 24px;
+            height: 24px;
+            margin-right: 12px;
+            border-radius: 50%;
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 12px;
+            font-weight: bold;
+        }
+        .status-text {
+            flex-grow: 1;
+        }
+        .status-time {
+            font-size: 0.8rem;
+            color: #9ca3af;
+            margin-left: 8px;
+        }
+        .status-pending {
+            background-color: #4a5568;
+        }
+        .status-processing {
+            background-color: #4299e1;
+            animation: pulse 1.5s infinite;
+        }
+        .status-complete {
+            background-color: #48bb78;
+        }
+        @keyframes pulse {
+            0% { opacity: 0.6; }
+            50% { opacity: 1; }
+            100% { opacity: 0.6; }
+        }
+        /* Custom scrollbar for insights panel */
+        .insights-panel::-webkit-scrollbar {
+            width: 8px;
+        }
+        .insights-panel::-webkit-scrollbar-track {
+            background: #1a1a1a;
+            border-radius: 4px;
+        }
+        .insights-panel::-webkit-scrollbar-thumb {
+            background: #4a4a4a;
+            border-radius: 4px;
+        }
+        .insights-panel::-webkit-scrollbar-thumb:hover {
+            background: #555;
+        }
+    </style>
 </head>
-<body class="bg-gray-100 min-h-screen">
+<body class="bg-gray-900 text-gray-100 min-h-screen">
     <div class="container mx-auto px-4 py-8">
-        <h1 class="text-4xl font-bold text-center text-blue-600 mb-8">Patent Explorer</h1>
+        <h1 class="text-4xl font-bold text-center text-blue-400 mb-8">Patent Technology Landscape</h1>
         
         <!-- Search Form -->
         <div class="max-w-2xl mx-auto mb-8">
-            <form id="searchForm" class="bg-white shadow-md rounded px-8 pt-6 pb-8 mb-4">
+            <form id="searchForm" class="bg-gray-800 shadow-lg rounded px-8 pt-6 pb-8 mb-4">
                 <div class="mb-4">
                     <input type="text" id="keywords" name="keywords" 
-                           class="shadow appearance-none border rounded w-full py-2 px-3 text-gray-700 leading-tight focus:outline-none focus:shadow-outline" 
-                           placeholder="Enter keywords to search patents...">
+                           class="w-full bg-gray-700 text-white rounded border border-gray-600 focus:border-blue-500 focus:ring-2 focus:ring-blue-900 py-2 px-4" 
+                           placeholder="Enter keywords to explore patent landscape...">
                 </div>
                 <div class="flex items-center justify-center">
                     <button type="submit" 
-                            class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded focus:outline-none focus:shadow-outline">
-                        Search Patents
+                            class="bg-blue-600 hover:bg-blue-700 text-white font-bold py-2 px-6 rounded focus:outline-none focus:shadow-outline transform hover:scale-105 transition-transform duration-200">
+                        Explore
                     </button>
                 </div>
             </form>
         </div>
 
-        <!-- Loading Spinner -->
-        <div id="loading" class="hidden">
-            <div class="flex justify-center items-center">
-                <div class="animate-spin rounded-full h-12 w-12 border-b-2 border-blue-500"></div>
+        <!-- Loading Status -->
+        <div id="loading" class="loading-container hidden">
+            <div class="mb-4">
+                <div class="flex justify-between items-center mb-2">
+                    <span class="text-sm font-medium" id="progress-text">Initializing...</span>
+                    <span class="text-sm font-medium" id="progress-percentage">0%</span>
+                </div>
+                <div class="progress-bar">
+                    <div class="progress-fill" id="progress-fill"></div>
+                </div>
+            </div>
+            <div class="status-list">
+                <div class="status-item" data-step="search">
+                    <div class="status-icon status-pending">1</div>
+                    <div class="status-text">Searching for patents</div>
+                    <div class="status-time"></div>
+                </div>
+                <div class="status-item" data-step="embedding">
+                    <div class="status-icon status-pending">2</div>
+                    <div class="status-text">Generating patent embeddings</div>
+                    <div class="status-time"></div>
+                </div>
+                <div class="status-item" data-step="clustering">
+                    <div class="status-icon status-pending">3</div>
+                    <div class="status-text">Identifying technology clusters</div>
+                    <div class="status-time"></div>
+                </div>
+                <div class="status-item" data-step="analysis">
+                    <div class="status-icon status-pending">4</div>
+                    <div class="status-text">Analyzing innovation opportunities</div>
+                    <div class="status-time"></div>
+                </div>
+                <div class="status-item" data-step="visualization">
+                    <div class="status-icon status-pending">5</div>
+                    <div class="status-text">Creating interactive visualization</div>
+                    <div class="status-time"></div>
+                </div>
             </div>
         </div>
 
-        <!-- Results Container -->
-        <div id="results" class="max-w-4xl mx-auto"></div>
+        <!-- Visualization Container -->
+        <div id="visualization" class="visualization-container"></div>
+
+        <!-- Insights Panel -->
+        <div id="insights" class="insights-panel p-4"></div>
     </div>
 
     <script>
+        let progressEventSource = null;
+
+        function startProgressMonitoring() {
+            if (progressEventSource) {
+                progressEventSource.close();
+            }
+            
+            progressEventSource = new EventSource('/progress');
+            progressEventSource.onmessage = function(event) {
+                const data = JSON.parse(event.data);
+                updateProgress(data.step, data.status);
+            };
+            
+            progressEventSource.onerror = function() {
+                progressEventSource.close();
+            };
+        }
+
+        function stopProgressMonitoring() {
+            if (progressEventSource) {
+                progressEventSource.close();
+                progressEventSource = null;
+            }
+        }
+
+        function updateProgress(step, status) {
+            const steps = ['search', 'embedding', 'clustering', 'analysis', 'visualization'];
+            const stepIndex = steps.indexOf(step);
+            const progress = ((stepIndex + 1) / steps.length) * 100;
+            
+            // Update progress bar
+            $('#progress-fill').css('width', `${progress}%`);
+            $('#progress-percentage').text(`${Math.round(progress)}%`);
+            
+            // Update status text
+            const statusTexts = {
+                'search': 'Searching patent database...',
+                'embedding': 'Generating patent embeddings...',
+                'clustering': 'Identifying technology clusters...',
+                'analysis': 'Analyzing innovation opportunities...',
+                'visualization': 'Creating interactive visualization...'
+            };
+            $('#progress-text').text(statusTexts[step]);
+            
+            // Update status items
+            steps.forEach((s, i) => {
+                const item = $(`.status-item[data-step="${s}"]`);
+                const time = item.find('.status-time');
+                
+                if (i < stepIndex) {
+                    item.removeClass('active').addClass('complete');
+                    if (!time.text()) {
+                        time.text(new Date().toLocaleTimeString());
+                    }
+                } else if (i === stepIndex) {
+                    item.addClass('active').removeClass('complete');
+                    time.text('In progress...');
+                } else {
+                    item.removeClass('active complete');
+                    time.text('');
+                }
+            });
+        }
+
         $(document).ready(function() {
             $('#searchForm').on('submit', function(e) {
                 e.preventDefault();
@@ -50,54 +279,102 @@
                     return;
                 }
 
-                // Show loading spinner
+                // Reset and show loading status
                 $('#loading').removeClass('hidden');
-                $('#results').empty();
+                $('#visualization').empty();
+                $('#insights').empty();
+                $('.progress-fill').css('width', '0%');
+                $('#progress-percentage').text('0%');
+                $('#progress-text').text('Initializing...');
+                $('.status-item').removeClass('active complete');
+                $('.status-time').text('');
+                
+                // Start progress monitoring
+                startProgressMonitoring();
 
                 $.ajax({
                     url: '/search',
                     method: 'POST',
                     data: { keywords: keywords },
                     success: function(response) {
-                        $('#loading').addClass('hidden');
-                        
                         if (response.error) {
-                            $('#results').html(`<div class="text-red-500 text-center">${response.error}</div>`);
+                            stopProgressMonitoring();
+                            $('#loading').addClass('hidden');
+                            alert(response.error);
                             return;
                         }
 
-                        if (!response.patents.length) {
-                            $('#results').html('<div class="text-center text-gray-600">No patents found.</div>');
-                            return;
+                        // Display visualization
+                        if (response.visualization) {
+                            const vizData = JSON.parse(response.visualization);
+                            Plotly.newPlot('visualization', vizData.data, vizData.layout);
+                            
+                            document.getElementById('visualization').on('plotly_click', function(data) {
+                                const link = data.points[0].customdata;
+                                if (link) {
+                                    window.open(link, '_blank');
+                                }
+                            });
                         }
 
-                        const resultsHtml = response.patents.map((patent, index) => `
-                            <div class="bg-white shadow-md rounded-lg p-6 mb-4">
-                                <h2 class="text-xl font-bold text-blue-600 mb-2">
-                                    <a href="${patent.link}" target="_blank" class="hover:underline">
-                                        ${patent.title}
-                                    </a>
-                                </h2>
-                                <div class="grid grid-cols-2 gap-4 mb-4 text-sm">
-                                    <div>
-                                        <span class="font-semibold">Assignee:</span> ${patent.assignee}
-                                    </div>
-                                    <div>
-                                        <span class="font-semibold">Filing Year:</span> ${patent.filing_year}
-                                    </div>
-                                </div>
-                                <div class="text-gray-600">
-                                    <span class="font-semibold">Abstract:</span><br>
-                                    ${patent.abstract}
-                                </div>
-                            </div>
-                        `).join('');
+                        // Display insights with two-column layout
+                        if (response.insights) {
+                            let insightsHtml = '<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 p-6">';
+                            
+                            // Left column: Innovation Opportunities
+                            insightsHtml += '<div class="col-span-1">';
+                            insightsHtml += '<h3 class="text-2xl font-bold mb-4 text-green-400">Innovation Opportunities</h3>';
+                            
+                            const opportunityZones = response.insights.filter(i => i.type === 'opportunity_zone');
+                            if (opportunityZones.length > 0) {
+                                insightsHtml += '<div class="space-y-4">';
+                                opportunityZones.forEach(zone => {
+                                    insightsHtml += `
+                                        <div class="opportunity-card p-6 text-base">
+                                            <div class="text-green-300 text-lg font-bold mb-3">Sparse Region (${zone.size} patents)</div>
+                                            <div class="text-gray-300 whitespace-pre-line leading-relaxed">${zone.description}</div>
+                                        </div>
+                                    `;
+                                });
+                                insightsHtml += '</div>';
+                            } else {
+                                insightsHtml += '<p class="text-gray-400">No innovation opportunities identified.</p>';
+                            }
+                            insightsHtml += '</div>';
+
+                            // Right column: Technology Clusters
+                            insightsHtml += '<div class="col-span-1">';
+                            insightsHtml += '<h3 class="text-2xl font-bold mb-4 text-blue-400">Technology Clusters</h3>';
+                            
+                            const clusters = response.insights.filter(i => i.type === 'cluster');
+                            if (clusters.length > 0) {
+                                insightsHtml += '<div class="space-y-4">';
+                                clusters.forEach(cluster => {
+                                    insightsHtml += `
+                                        <div class="cluster-card p-6 text-base">
+                                            <div class="text-blue-300 text-lg font-bold mb-3">Cluster ${cluster.id} (${cluster.size} patents)</div>
+                                            <div class="text-gray-300 whitespace-pre-line leading-relaxed">${cluster.description}</div>
+                                        </div>
+                                    `;
+                                });
+                                insightsHtml += '</div>';
+                            } else {
+                                insightsHtml += '<p class="text-gray-400">No technology clusters identified.</p>';
+                            }
+                            insightsHtml += '</div>';
 
-                        $('#results').html(resultsHtml);
+                            insightsHtml += '</div>';
+                            $('#insights').html(insightsHtml);
+                        }
+
+                        // Stop progress monitoring and hide loading status
+                        stopProgressMonitoring();
+                        $('#loading').addClass('hidden');
                     },
                     error: function() {
+                        stopProgressMonitoring();
                         $('#loading').addClass('hidden');
-                        $('#results').html('<div class="text-red-500 text-center">An error occurred while searching patents.</div>');
+                        alert('An error occurred while analyzing patents.');
                     }
                 });
             });