Implement chunked processing for player similarity score calculation to enhance memory efficiency in predict_dupes.py

global_func/predict_dupes.py

@@ -52,53 +52,59 @@ def calculate_weighted_ownership_wrapper(row_ownerships):
     ownership_array = row_ownerships.values.reshape(1, -1)
     return calculate_weighted_ownership_vectorized(ownership_array)[0]
 
-def calculate_player_similarity_score_vectorized(portfolio, player_columns):
+def calculate_player_similarity_score_chunked(portfolio, player_columns, chunk_size=1000):
     """
-    Vectorized version of calculate_player_similarity_score using NumPy operations.
+    Memory-efficient version that processes similarities in chunks
     """
-    # Extract player data and convert to string array
+    # Same setup as before
     player_data = portfolio[player_columns].astype(str).fillna('').values
     
-    # Get all unique players and create a mapping to numeric IDs
     all_players = set()
     for row in player_data:
         for val in row:
             if isinstance(val, str) and val.strip() != '':
                 all_players.add(val)
     
-    # Create player ID mapping
     player_to_id = {player: idx for idx, player in enumerate(sorted(all_players))}
     
-    # Convert each row to a binary vector (1 if player is present, 0 if not)
     n_players = len(all_players)
     n_rows = len(portfolio)
     binary_matrix = np.zeros((n_rows, n_players), dtype=np.int8)
     
-    # Vectorized binary matrix creation
     for i, row in enumerate(player_data):
         for val in row:
             if isinstance(val, str) and str(val).strip() != '' and str(val) in player_to_id:
                 binary_matrix[i, player_to_id[str(val)]] = 1
     
-    # Vectorized Jaccard distance calculation
-    intersection_matrix = np.dot(binary_matrix, binary_matrix.T)
-    row_sums = np.sum(binary_matrix, axis=1)
-    union_matrix = row_sums[:, np.newaxis] + row_sums - intersection_matrix
+    # Process similarities in chunks to avoid massive matrices
+    similarity_scores = np.zeros(n_rows)
     
-    # Calculate Jaccard distance: 1 - (intersection / union)
-    with np.errstate(divide='ignore', invalid='ignore'):
-        jaccard_similarity = np.divide(intersection_matrix, union_matrix, 
-                                     out=np.zeros_like(intersection_matrix, dtype=float), 
-                                     where=union_matrix != 0)
-    
-    jaccard_distance = 1 - jaccard_similarity
-    
-    # Exclude self-comparison and calculate average distance for each row
-    np.fill_diagonal(jaccard_distance, 0)
-    row_counts = n_rows - 1
-    similarity_scores = np.sum(jaccard_distance, axis=1) / row_counts
+    for i in range(0, n_rows, chunk_size):
+        end_i = min(i + chunk_size, n_rows)
+        chunk_binary = binary_matrix[i:end_i]
+        
+        # Calculate similarities for this chunk only
+        intersection = np.dot(chunk_binary, binary_matrix.T)
+        chunk_row_sums = np.sum(chunk_binary, axis=1)
+        all_row_sums = np.sum(binary_matrix, axis=1)
+        
+        union = chunk_row_sums[:, np.newaxis] + all_row_sums - intersection
+        
+        with np.errstate(divide='ignore', invalid='ignore'):
+            jaccard_sim = np.divide(intersection, union, 
+                                  out=np.zeros_like(intersection, dtype=float), 
+                                  where=union != 0)
+        
+        jaccard_dist = 1 - jaccard_sim
+        
+        # Exclude self-comparison and calculate average
+        for j in range(len(jaccard_dist)):
+            actual_idx = i + j
+            jaccard_dist[j, actual_idx] = 0  # Exclude self
+            
+        similarity_scores[i:end_i] = np.sum(jaccard_dist, axis=1) / (n_rows - 1)
     
-    # Normalize to 0-1 scale
+    # Normalize
     score_range = similarity_scores.max() - similarity_scores.min()
     if score_range > 0:
         similarity_scores = (similarity_scores - similarity_scores.min()) / score_range