Add memory-efficient ranking calculation in predict_dupes.py

Introduced a new function, calculate_flex_ranks_efficient, to replace pd.concat and rank operations, enhancing memory efficiency. Updated the predict_dupes function to utilize this new ranking method for FLEX and position ownership percent ranks.

global_func/predict_dupes.py

@@ -4,6 +4,7 @@ import pandas as pd
 import time
 import math
 from difflib import SequenceMatcher
+import scipy.stats
 
 def calculate_weighted_ownership_vectorized(ownership_array):
     """
@@ -38,6 +39,32 @@ def calculate_weighted_ownership_vectorized(ownership_array):
     # Convert back to percentage form
     return weighted * 10000
 
+def calculate_flex_ranks_efficient(portfolio, start_col, end_col, maps_dict, map_key='own_map'):
+    """Memory-efficient replacement for pd.concat + rank operations"""
+    n_rows = len(portfolio)
+    n_cols = end_col - start_col
+    
+    # Pre-allocate result arrays
+    all_values = np.zeros(n_rows * n_cols, dtype=np.float32)
+    
+    # Fill values column by column
+    for i, col_idx in enumerate(range(start_col, end_col)):
+        start_idx = i * n_rows
+        end_idx = (i + 1) * n_rows
+        all_values[start_idx:end_idx] = portfolio.iloc[:, col_idx].map(maps_dict[map_key]).values
+    
+    # Calculate percentile ranks efficiently
+    ranks = scipy.stats.rankdata(all_values, method='average') / len(all_values)
+    
+    # Reshape back to individual column ranks
+    result_ranks = {}
+    for i in range(n_cols):
+        start_idx = i * n_rows
+        end_idx = (i + 1) * n_rows
+        result_ranks[i] = ranks[start_idx:end_idx]
+    
+    return result_ranks
+
 def calculate_weighted_ownership_wrapper(row_ownerships):
     """
     Wrapper function for the original calculate_weighted_ownership to work with Pandas .apply()
@@ -144,22 +171,14 @@ def predict_dupes(portfolio, maps_dict, site_var, type_var, Contest_Size, streng
         
         # Assign ranks back to individual columns using the same rank scale
         if sport_var == 'GOLF':
-            flex_ownerships = pd.concat([
-                portfolio.iloc[:,1].map(maps_dict['own_map']),
-                portfolio.iloc[:,2].map(maps_dict['own_map']),
-                portfolio.iloc[:,3].map(maps_dict['own_map']),
-                portfolio.iloc[:,4].map(maps_dict['own_map']),
-                portfolio.iloc[:,5].map(maps_dict['own_map']),
-                portfolio.iloc[:,6].map(maps_dict['own_map'])
-            ])
-            flex_rank = flex_ownerships.rank(pct=True)
+            flex_ranks = calculate_flex_ranks_efficient(portfolio, 1, 7, maps_dict)
 
-            portfolio['FLEX1_Own_percent_rank'] = flex_rank.iloc[0:n_rows].values
-            portfolio['FLEX2_Own_percent_rank'] = flex_rank.iloc[n_rows:2*n_rows].values  
-            portfolio['FLEX3_Own_percent_rank'] = flex_rank.iloc[2*n_rows:3*n_rows].values
-            portfolio['FLEX4_Own_percent_rank'] = flex_rank.iloc[3*n_rows:4*n_rows].values
-            portfolio['FLEX5_Own_percent_rank'] = flex_rank.iloc[4*n_rows:5*n_rows].values
-            portfolio['FLEX6_Own_percent_rank'] = flex_rank.iloc[5*n_rows:6*n_rows].values
+            portfolio['FLEX1_Own_percent_rank'] = flex_ranks[0]
+            portfolio['FLEX2_Own_percent_rank'] = flex_ranks[1]  
+            portfolio['FLEX3_Own_percent_rank'] = flex_ranks[2]
+            portfolio['FLEX4_Own_percent_rank'] = flex_ranks[3]
+            portfolio['FLEX5_Own_percent_rank'] = flex_ranks[4]
+            portfolio['FLEX6_Own_percent_rank'] = flex_ranks[5]
 
             portfolio['FLEX1_Own'] = portfolio.iloc[:,0].map(maps_dict['own_map']).astype('float32') / 100
             portfolio['FLEX2_Own'] = portfolio.iloc[:,1].map(maps_dict['own_map']).astype('float32') / 100
@@ -168,21 +187,14 @@ def predict_dupes(portfolio, maps_dict, site_var, type_var, Contest_Size, streng
             portfolio['FLEX5_Own'] = portfolio.iloc[:,4].map(maps_dict['own_map']).astype('float32') / 100
             portfolio['FLEX6_Own'] = portfolio.iloc[:,5].map(maps_dict['own_map']).astype('float32') / 100
         else:
-            flex_ownerships = pd.concat([
-                portfolio.iloc[:,1].map(maps_dict['own_map']),
-                portfolio.iloc[:,2].map(maps_dict['own_map']),
-                portfolio.iloc[:,3].map(maps_dict['own_map']),
-                portfolio.iloc[:,4].map(maps_dict['own_map']),
-                portfolio.iloc[:,5].map(maps_dict['own_map'])
-            ])
-            flex_rank = flex_ownerships.rank(pct=True)
+            flex_ranks = calculate_flex_ranks_efficient(portfolio, 1, 6, maps_dict)
             
             portfolio['CPT_Own_percent_rank'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).rank(pct=True)
-            portfolio['FLEX1_Own_percent_rank'] = flex_rank.iloc[0:n_rows].values
-            portfolio['FLEX2_Own_percent_rank'] = flex_rank.iloc[n_rows:2*n_rows].values  
-            portfolio['FLEX3_Own_percent_rank'] = flex_rank.iloc[2*n_rows:3*n_rows].values
-            portfolio['FLEX4_Own_percent_rank'] = flex_rank.iloc[3*n_rows:4*n_rows].values
-            portfolio['FLEX5_Own_percent_rank'] = flex_rank.iloc[4*n_rows:5*n_rows].values
+            portfolio['FLEX1_Own_percent_rank'] = flex_ranks[0]
+            portfolio['FLEX2_Own_percent_rank'] = flex_ranks[1]  
+            portfolio['FLEX3_Own_percent_rank'] = flex_ranks[2]
+            portfolio['FLEX4_Own_percent_rank'] = flex_ranks[3]
+            portfolio['FLEX5_Own_percent_rank'] = flex_ranks[4]
 
             portfolio['CPT_Own'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).astype('float32') / 100
             portfolio['FLEX1_Own'] = portfolio.iloc[:,1].map(maps_dict['own_map']).astype('float32') / 100
@@ -223,22 +235,15 @@ def predict_dupes(portfolio, maps_dict, site_var, type_var, Contest_Size, streng
 
             n_rows = len(portfolio)
 
-            flex_ownerships = pd.concat([
-                portfolio.iloc[:,1].map(maps_dict['own_map']),
-                portfolio.iloc[:,2].map(maps_dict['own_map']),
-                portfolio.iloc[:,3].map(maps_dict['own_map']),
-                portfolio.iloc[:,4].map(maps_dict['own_map']),
-                portfolio.iloc[:,5].map(maps_dict['own_map'])
-            ])
-            flex_rank = flex_ownerships.rank(pct=True)
+            flex_ranks = calculate_flex_ranks_efficient(portfolio, 1, 6, maps_dict)
 
             # Assign ranks back to individual columns using the same rank scale
             portfolio['CPT_Own_percent_rank'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).rank(pct=True)
-            portfolio['FLEX1_Own_percent_rank'] = flex_rank.iloc[0:n_rows].values
-            portfolio['FLEX2_Own_percent_rank'] = flex_rank.iloc[n_rows:2*n_rows].values  
-            portfolio['FLEX3_Own_percent_rank'] = flex_rank.iloc[2*n_rows:3*n_rows].values
-            portfolio['FLEX4_Own_percent_rank'] = flex_rank.iloc[3*n_rows:4*n_rows].values
-            portfolio['FLEX5_Own_percent_rank'] = flex_rank.iloc[4*n_rows:5*n_rows].values
+            portfolio['FLEX1_Own_percent_rank'] = flex_ranks[0]
+            portfolio['FLEX2_Own_percent_rank'] = flex_ranks[1]  
+            portfolio['FLEX3_Own_percent_rank'] = flex_ranks[2]
+            portfolio['FLEX4_Own_percent_rank'] = flex_ranks[3]
+            portfolio['FLEX5_Own_percent_rank'] = flex_ranks[4]
 
             portfolio['CPT_Own'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).astype('float32') / 100
             portfolio['FLEX1_Own'] = portfolio.iloc[:,1].map(maps_dict['own_map']).astype('float32') / 100
@@ -276,24 +281,16 @@ def predict_dupes(portfolio, maps_dict, site_var, type_var, Contest_Size, streng
             
             n_rows = len(portfolio)
 
-            flex_ownerships = pd.concat([
-                portfolio.iloc[:,1].map(maps_dict['own_map']),
-                portfolio.iloc[:,2].map(maps_dict['own_map']),
-                portfolio.iloc[:,3].map(maps_dict['own_map']),
-                portfolio.iloc[:,4].map(maps_dict['own_map']),
-                portfolio.iloc[:,5].map(maps_dict['own_map']),
-                portfolio.iloc[:,6].map(maps_dict['own_map'])
-            ])
-            flex_rank = flex_ownerships.rank(pct=True)
+            flex_ranks = calculate_flex_ranks_efficient(portfolio, 1, 7, maps_dict)
             
             # Assign ranks back to individual columns using the same rank scale
             portfolio['CPT_Own_percent_rank'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).rank(pct=True)
-            portfolio['TOP_Own_percent_rank'] = flex_rank.iloc[0:n_rows].values
-            portfolio['JNG_Own_percent_rank'] = flex_rank.iloc[n_rows:2*n_rows].values
-            portfolio['MID_Own_percent_rank'] = flex_rank.iloc[2*n_rows:3*n_rows].values
-            portfolio['ADC_Own_percent_rank'] = flex_rank.iloc[3*n_rows:4*n_rows].values
-            portfolio['SUP_Own_percent_rank'] = flex_rank.iloc[4*n_rows:5*n_rows].values
-            portfolio['Team_Own_percent_rank'] = flex_rank.iloc[5*n_rows:6*n_rows].values
+            portfolio['TOP_Own_percent_rank'] = flex_ranks[0]
+            portfolio['JNG_Own_percent_rank'] = flex_ranks[1]
+            portfolio['MID_Own_percent_rank'] = flex_ranks[2]
+            portfolio['ADC_Own_percent_rank'] = flex_ranks[3]
+            portfolio['SUP_Own_percent_rank'] = flex_ranks[4]
+            portfolio['Team_Own_percent_rank'] = flex_ranks[5]
 
             portfolio['CPT_Own'] = portfolio.iloc[:,0].map(maps_dict['cpt_own_map']).astype('float32') / 100
             portfolio['TOP_Own'] = portfolio.iloc[:,1].map(maps_dict['own_map']).astype('float32') / 100