Update app.py

app.py

@@ -115,6 +115,7 @@ def extract_frames(video_path, output_folder, desired_fps, progress_callback=Non
 def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, batch_size):
     embeddings_by_frame = {}
     emotions_by_frame = {}
+    aligned_face_paths = []
     frame_files = sorted([f for f in os.listdir(frames_folder) if f.endswith('.jpg')])
 
     for i in range(0, len(frame_files), batch_size):
@@ -144,6 +145,7 @@ def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, b
                             aligned_face_resized = cv2.resize(aligned_face, (160, 160))
                             output_path = os.path.join(aligned_faces_folder, f"frame_{frame_num}_face.jpg")
                             cv2.imwrite(output_path, aligned_face_resized)
+                            aligned_face_paths.append(output_path)
                             embedding, emotion = get_face_embedding_and_emotion(aligned_face_resized)
                             embeddings_by_frame[frame_num] = embedding
                             emotions_by_frame[frame_num] = emotion
@@ -151,35 +153,33 @@ def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, b
         progress((i + len(batch_files)) / frame_count,
                  f"Processing frames {i + 1} to {min(i + len(batch_files), frame_count)} of {frame_count}")
 
-    return embeddings_by_frame, emotions_by_frame
+    return embeddings_by_frame, emotions_by_frame, aligned_face_paths
 
+def cluster_faces(face_images):
+    if len(face_images) < 2:
+        print("Not enough faces for clustering. Assigning all to one cluster.")
+        return np.zeros(len(face_images), dtype=int)
 
-def cluster_embeddings(embeddings):
-    if len(embeddings) < 2:
-        print("Not enough embeddings for clustering. Assigning all to one cluster.")
-        return np.zeros(len(embeddings), dtype=int)
+    # Resize all images to a consistent size
+    resized_faces = [cv2.resize(face, (224, 224)) for face in face_images]
 
-    scaler = StandardScaler()
-    embeddings_scaled = scaler.fit_transform(embeddings)
+    # Convert images to grayscale and flatten
+    gray_faces = [cv2.cvtColor(face, cv2.COLOR_BGR2GRAY).flatten() for face in resized_faces]
 
-    # Use DBSCAN for adaptive clustering
-    dbscan = DBSCAN(eps=0.5, min_samples=5)  # Adjust these parameters as needed
-    clusters = dbscan.fit_predict(embeddings_scaled)
+    # Stack the flattened images
+    X = np.stack(gray_faces)
 
-    # If DBSCAN couldn't find meaningful clusters, fall back to KMeans
-    if len(set(clusters)) == 1:
-        best_n_clusters = 1
-        best_score = -1
-        for n_clusters in range(2, min(5, len(embeddings))):
-            kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
-            labels = kmeans.fit_predict(embeddings_scaled)
-            score = silhouette_score(embeddings_scaled, labels)
-            if score > best_score:
-                best_score = score
-                best_n_clusters = n_clusters
+    # Normalize the pixel values
+    X = X / 255.0
 
-        kmeans = KMeans(n_clusters=best_n_clusters, random_state=42, n_init=10)
-        clusters = kmeans.fit_predict(embeddings_scaled)
+    # Perform DBSCAN clustering
+    dbscan = DBSCAN(eps=0.3, min_samples=3, metric='euclidean')
+    clusters = dbscan.fit_predict(X)
+
+    # If DBSCAN assigns all to noise (-1), consider it as one cluster
+    if np.all(clusters == -1):
+        print("DBSCAN assigned all to noise. Considering as one cluster.")
+        return np.zeros(len(face_images), dtype=int)
 
     return clusters
 
@@ -310,38 +310,41 @@ def lstm_anomaly_detection(X, feature_columns, num_anomalies=10, epochs=100, bat
             anomalies_comp, mse_comp, top_indices_comp,
             model)
 
-from scipy import interpolate
+def emotion_anomaly_detection(emotion_data, num_anomalies=10, epochs=100, batch_size=64):
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    X = torch.FloatTensor(emotion_data.values.reshape(-1, 1)).to(device)
+    X = X.unsqueeze(0)  # Add batch dimension
+
+    model = LSTMAutoencoder(input_size=1).to(device)
+    criterion = nn.MSELoss()
+    optimizer = optim.Adam(model.parameters())
+
+    for epoch in range(epochs):
+        model.train()
+        optimizer.zero_grad()
+        output = model(X)
+        loss = criterion(output, X)
+        loss.backward()
+        optimizer.step()
+
+    model.eval()
+    with torch.no_grad():
+        reconstructed = model(X).squeeze(0).cpu().numpy()
+
+    mse = np.mean(np.power(X.squeeze(0).cpu().numpy() - reconstructed, 2), axis=1)
+    top_indices = mse.argsort()[-num_anomalies:][::-1]
+    anomalies = np.zeros(len(mse), dtype=bool)
+    anomalies[top_indices] = True
+
+    return anomalies, mse, top_indices
+
+def normalize_scores(scores):
+    min_score = np.min(scores)
+    max_score = np.max(scores)
+    if max_score == min_score:
+        return np.full_like(scores, 100)
+    return ((scores - min_score) / (max_score - min_score)) * 100
 
-def plot_with_segments(ax, df_filtered, y_column, color):
-    segments = []
-    current_segment = []
-    for i, (time, score) in enumerate(zip(df_filtered['Seconds'], df_filtered[y_column])):
-        if i > 0 and time - df_filtered['Seconds'].iloc[i-1] > 1:  # Gap of more than 1 second
-            if current_segment:
-                segments.append(current_segment)
-                current_segment = []
-        current_segment.append((time, score))
-    if current_segment:
-        segments.append(current_segment)
-
-    for segment in segments:
-        times, scores = zip(*segment)
-        if len(times) > 3:
-            try:
-                # Use scipy's interpolate to create a smooth curve
-                f = interpolate.interp1d(times, scores, kind='cubic')
-                smooth_times = np.linspace(min(times), max(times), num=200)
-                smooth_scores = f(smooth_times)
-                ax.plot(smooth_times, smooth_scores, color=color, linewidth=1.5)
-            except ValueError:
-                # Fall back to linear interpolation if cubic fails
-                f = interpolate.interp1d(times, scores, kind='linear')
-                smooth_times = np.linspace(min(times), max(times), num=200)
-                smooth_scores = f(smooth_times)
-                ax.plot(smooth_times, smooth_scores, color=color, linewidth=1.5)
-        else:
-            # For very short segments, just plot the points
-            ax.plot(times, scores, color=color, linewidth=1.5)
 
 def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
     plt.figure(figsize=(16, 8), dpi=400)
@@ -350,23 +353,21 @@ def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
     df['Seconds'] = df['Timecode'].apply(
         lambda x: sum(float(t) * 60 ** i for i, t in enumerate(reversed(x.split(':')))))
 
-    # Filter out rows with no data
-    mask = ~np.isnan(anomaly_scores)
-    df_filtered = df[mask].copy()
-    df_filtered['anomaly_scores'] = anomaly_scores[mask]
+    # Normalize scores
+    normalized_scores = normalize_scores(anomaly_scores)
 
-    if df_filtered.empty:
-        ax.text(0.5, 0.5, "No data available", ha='center', va='center')
-    else:
-        plot_with_segments(ax, df_filtered, 'anomaly_scores', 'blue')
+    # Omit the first data point
+    seconds = df['Seconds'].values[1:]
+    scores = normalized_scores[1:]
+
+    # Create bar plot
+    ax.bar(seconds, scores, width=1, color='blue', alpha=0.7)
 
-        # Highlight top anomalies
-        top_indices_filtered = [i for i in top_indices if i in df_filtered.index]
-        ax.scatter(df_filtered['Seconds'].iloc[top_indices_filtered],
-                   df_filtered['anomaly_scores'].iloc[top_indices_filtered],
-                   color='red', s=100, zorder=5)
+    # Highlight top anomalies (excluding the first data point)
+    top_indices = [idx for idx in top_indices if idx > 0]
+    ax.bar(df['Seconds'].iloc[top_indices], normalized_scores[top_indices], width=1, color='red', alpha=0.7)
 
-    max_seconds = df['Seconds'].max()  # Use the full range for x-axis
+    max_seconds = df['Seconds'].max()
     ax.set_xlim(0, max_seconds)
     num_ticks = 80
     ax.set_xticks(np.linspace(0, max_seconds, num_ticks))
@@ -375,35 +376,32 @@ def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
 
     ax.set_xlabel('Time')
     ax.set_ylabel('Anomaly Score')
-    ax.set_title(f'Anomaly Scores Over Time ({title})')
+    ax.set_title(f'Anomaly Scores ({title})')
 
     ax.grid(True, linestyle='--', alpha=0.7)
     plt.tight_layout()
     return fig
 
-def plot_emotion(df, emotion, num_anomalies, color):
+
+def plot_emotion(df, emotion, anomaly_scores, top_indices, num_anomalies, color):
     plt.figure(figsize=(16, 8), dpi=400)
     fig, ax = plt.subplots(figsize=(16, 8))
 
     df['Seconds'] = df['Timecode'].apply(
         lambda x: sum(float(t) * 60 ** i for i, t in enumerate(reversed(x.split(':')))))
 
-    # Filter out rows with no data
-    mask = ~np.isnan(df[emotion])
-    df_filtered = df[mask]
+    # Omit the first data point
+    seconds = df['Seconds'].values[1:]
+    scores = anomaly_scores[1:]
 
-    if df_filtered.empty:
-        ax.text(0.5, 0.5, "No data available", ha='center', va='center')
-    else:
-        plot_with_segments(ax, df_filtered, emotion, color)
+    # Create bar plot
+    ax.bar(seconds, scores, width=1, color=color, alpha=0.7)
 
-        # Highlight top anomalies
-        top_indices = np.argsort(df_filtered[emotion].values)[-num_anomalies:][::-1]
-        ax.scatter(df_filtered['Seconds'].iloc[top_indices],
-                   df_filtered[emotion].iloc[top_indices],
-                   color='red', s=100, zorder=5)
+    # Highlight top anomalies (excluding the first data point)
+    top_indices = [idx for idx in top_indices if idx > 0]
+    ax.bar(df['Seconds'].iloc[top_indices], anomaly_scores[top_indices], width=1, color='red', alpha=0.7)
 
-    max_seconds = df['Seconds'].max()  # Use the full range for x-axis
+    max_seconds = df['Seconds'].max()
     ax.set_xlim(0, max_seconds)
     num_ticks = 80
     ax.set_xticks(np.linspace(0, max_seconds, num_ticks))
@@ -411,30 +409,32 @@ def plot_emotion(df, emotion, num_anomalies, color):
                        rotation=90, ha='center', va='top')
 
     ax.set_xlabel('Time')
-    ax.set_ylabel(f'{emotion.capitalize()} Score')
-    ax.set_title(f'{emotion.capitalize()} Scores Over Time (Top {num_anomalies} in Red)')
+    ax.set_ylabel(f'{emotion.capitalize()} Anomaly Score')
+    ax.set_title(f'{emotion.capitalize()} Anomaly Scores (Top {num_anomalies} in Red)')
 
     ax.grid(True, linestyle='--', alpha=0.7)
     plt.tight_layout()
     return fig
 
+
 def get_random_face_samples(organized_faces_folder, output_folder):
-    face_samples = {}
+    face_samples = []
     for cluster_folder in os.listdir(organized_faces_folder):
         if cluster_folder.startswith("person_"):
-            cluster_id = int(cluster_folder.split("_")[1])
             person_folder = os.path.join(organized_faces_folder, cluster_folder)
             face_files = [f for f in os.listdir(person_folder) if f.endswith('.jpg')]
             if face_files:
                 random_face = np.random.choice(face_files)
                 face_path = os.path.join(person_folder, random_face)
-                output_path = os.path.join(output_folder, f"face_sample_person_{cluster_id}.jpg")
+                output_path = os.path.join(output_folder, f"face_sample_{cluster_folder}.jpg")
                 face_img = cv2.imread(face_path)
-                small_face = cv2.resize(face_img, (160, 160))
-                cv2.imwrite(output_path, small_face)
-                face_samples[cluster_id] = output_path
+                if face_img is not None:
+                    small_face = cv2.resize(face_img, (160, 160))
+                    cv2.imwrite(output_path, small_face)
+                    face_samples.append(output_path)
     return face_samples
 
+
 def process_video(video_path, num_anomalies, num_components, desired_fps, batch_size, progress=gr.Progress()):
     output_folder = "output"
     os.makedirs(output_folder, exist_ok=True)
@@ -459,16 +459,17 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
 
         progress(1, "Frame extraction complete")
         progress(0.3, "Processing frames")
-        embeddings_by_frame, emotions_by_frame = process_frames(frames_folder, aligned_faces_folder, frame_count,
-                                                                progress, batch_size)
+        embeddings_by_frame, emotions_by_frame, aligned_face_paths = process_frames(frames_folder, aligned_faces_folder,
+                                                                                    frame_count,
+                                                                                    progress, batch_size)
 
-        if not embeddings_by_frame:
+        if not aligned_face_paths:
             return ("No faces were extracted from the video.",
                     None, None, None, None, None, None, None, None, None)
 
-        progress(0.6, "Clustering embeddings")
-        embeddings = list(embeddings_by_frame.values())
-        clusters = cluster_embeddings(embeddings)
+        progress(0.6, "Clustering faces")
+        face_images = [cv2.imread(path) for path in aligned_face_paths]
+        clusters = cluster_faces(face_images)
         num_clusters = len(set(clusters))  # Get the number of unique clusters
 
         progress(0.7, "Organizing faces")
@@ -485,11 +486,25 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
         feature_columns = [col for col in df.columns if
                            col not in ['Frame', 'Timecode', 'Time (Minutes)', 'Embedding_Index']]
         X = df[feature_columns].values
-        print(f"Shape of input data: {X.shape}")
-        print(f"Feature columns: {feature_columns}")
+
         try:
             anomalies_all, anomaly_scores_all, top_indices_all, anomalies_comp, anomaly_scores_comp, top_indices_comp, _ = lstm_anomaly_detection(
                 X, feature_columns, num_anomalies=num_anomalies, batch_size=batch_size)
+
+            # Normalize anomaly scores
+            anomaly_scores_all = normalize_scores(anomaly_scores_all)
+            anomaly_scores_comp = normalize_scores(anomaly_scores_comp)
+
+            # Perform anomaly detection for each emotion
+            emotion_anomalies = {}
+            for emotion in ['fear', 'sad', 'angry', 'happy', 'surprise', 'neutral']:
+                anomalies, scores, indices = emotion_anomaly_detection(df[emotion], num_anomalies=num_anomalies)
+                emotion_anomalies[emotion] = {
+                    'anomalies': anomalies,
+                    'scores': normalize_scores(scores),
+                    'indices': indices
+                }
+
         except Exception as e:
             print(f"Error details: {str(e)}")
             return f"Error in anomaly detection: {str(e)}", None, None, None, None, None, None, None, None, None
@@ -499,38 +514,39 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
             anomaly_plot_all = plot_anomaly_scores(df, anomaly_scores_all, top_indices_all, "All Features")
             anomaly_plot_comp = plot_anomaly_scores(df, anomaly_scores_comp, top_indices_comp, "Components Only")
             emotion_plots = [
-                plot_emotion(df, 'fear', num_anomalies, 'purple'),
-                plot_emotion(df, 'sad', num_anomalies, 'green'),
-                plot_emotion(df, 'angry', num_anomalies, 'orange'),
-                plot_emotion(df, 'happy', num_anomalies, 'darkblue'),
-                plot_emotion(df, 'surprise', num_anomalies, 'gold'),
-                plot_emotion(df, 'neutral', num_anomalies, 'grey')
+                plot_emotion(df, emotion, emotion_anomalies[emotion]['scores'], emotion_anomalies[emotion]['indices'],
+                             num_anomalies, color)
+                for emotion, color in zip(['fear', 'sad', 'angry', 'happy', 'surprise', 'neutral'],
+                                          ['purple', 'green', 'orange', 'darkblue', 'gold', 'grey'])
             ]
         except Exception as e:
             return f"Error generating plots: {str(e)}", None, None, None, None, None, None, None, None, None
 
         progress(1.0, "Preparing results")
-        results = f"Number of persons detected: {num_clusters}\n\n"
+        results = f"Number of persons/clusters detected: {num_clusters}\n\n"
+        results += f"Breakdown of persons/clusters:\n"
         for cluster_id in range(num_clusters):
-            results += f"Person {cluster_id + 1}: {len([c for c in clusters if c == cluster_id])} frames\n"
+            results += f"Person/Cluster {cluster_id + 1}: {len([c for c in clusters if c == cluster_id])} frames\n"
         results += f"\nTop {num_anomalies} anomalies (All Features):\n"
-        results += "\n".join([f"{score:.4f} at {timecode}" for score, timecode in
-                              zip(anomaly_scores_all[top_indices_all], df['Timecode'].iloc[top_indices_all].values)])
+        results += "\n".join([f"{score:.2f} at {timecode}" for score, timecode in
+                              zip(anomaly_scores_all[top_indices_all[1:]],
+                                  df['Timecode'].iloc[top_indices_all[1:]].values)])
         results += f"\n\nTop {num_anomalies} anomalies (Components Only):\n"
-        results += "\n".join([f"{score:.4f} at {timecode}" for score, timecode in
-                              zip(anomaly_scores_comp[top_indices_comp], df['Timecode'].iloc[top_indices_comp].values)])
+        results += "\n".join([f"{score:.2f} at {timecode}" for score, timecode in
+                              zip(anomaly_scores_comp[top_indices_comp[1:]],
+                                  df['Timecode'].iloc[top_indices_comp[1:]].values)])
 
         for emotion in ['fear', 'sad', 'angry', 'happy', 'surprise', 'neutral']:
-            top_indices = np.argsort(df[emotion].values)[-num_anomalies:][::-1]
-            results += f"\n\nTop {num_anomalies} {emotion.capitalize()} Scores:\n"
-            results += "\n".join([f"{df[emotion].iloc[i]:.4f} at {df['Timecode'].iloc[i]}" for i in top_indices])
+            results += f"\n\nTop {num_anomalies} {emotion.capitalize()} Anomalies:\n"
+            results += "\n".join([f"{emotion_anomalies[emotion]['scores'][i]:.2f} at {df['Timecode'].iloc[i]}"
+                                  for i in emotion_anomalies[emotion]['indices'] if i > 0])
 
         return (
             results,
             anomaly_plot_all,
             anomaly_plot_comp,
             *emotion_plots,
-            *[face_samples.get(i, None) for i in range(num_clusters)]
+            face_samples
         )
 
 
@@ -541,7 +557,7 @@ iface = gr.Interface(
         gr.Slider(minimum=1, maximum=20, step=1, value=10, label="Number of Anomalies"),
         gr.Slider(minimum=1, maximum=20, step=1, value=10, label="Number of Components"),
         gr.Slider(minimum=1, maximum=20, step=1, value=15, label="Desired FPS"),
-        gr.Slider(minimum=1, maximum=32, step=4, value=8, label="Batch Size")
+        gr.Slider(minimum=1, maximum=32, step=1, value=8, label="Batch Size")
     ],
     outputs=[
         gr.Textbox(label="Anomaly Detection Results"),
@@ -557,15 +573,15 @@ iface = gr.Interface(
     ],
     title="Facial Expressions Anomaly Detection",
     description="""
-    This application detects anomalies in facial expressions and emotions from a video input. 
-    It identifies distinct persons in the video and provides a sample face for each.
-
-    Adjust the parameters as needed:
-    - Number of Anomalies: How many top anomalies or high intensities to highlight
-    - Number of Components: Complexity of the facial expression model
-    - Desired FPS: Frames per second to analyze (lower for faster processing)
-    - Batch Size: Affects processing speed and memory usage
-    """,
+        This application detects anomalies in facial expressions and emotions from a video input. 
+        It identifies distinct persons in the video and provides a sample face for each.
+
+        Adjust the parameters as needed:
+        - Number of Anomalies: How many top anomalies or high intensities to highlight
+        - Number of Components: Complexity of the facial expression model
+        - Desired FPS: Frames per second to analyze (lower for faster processing)
+        - Batch Size: Affects processing speed and memory usage
+        """,
     allow_flagging="never"
 )