Update src/main.py

src/main.py

@@ -0,0 +1,435 @@
+import os
+import logging
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, TensorDataset
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+from sklearn.model_selection import KFold
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.cluster import DBSCAN
+from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+from sklearn.base import BaseEstimator, TransformerMixin
+import dask.dataframe as dd
+import optuna
+import numpy as np
+import pandas as pd
+import pytorch_lightning as pl
+import json
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Constants
+RANDOM_SEED = 42
+N_SPLITS = 5
+
+# Define paths
+MODEL_CONFIG_PATH = 'model_config.json'
+BEST_MODEL_PATH = 'best_model.pt'
+FINAL_MODEL_PATH = 'final_model.pt'
+
+class AdvancedFeatureEngineering(BaseEstimator, TransformerMixin):
+    """Custom transformer for advanced feature engineering."""
+    def __init__(self, numeric_cols):
+        self.numeric_cols = numeric_cols
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X):
+        X = X.copy()
+        # Feature engineering for rank and elo differences with adjusted weighting
+        X['rank_difference'] = X['winner_rank'] - X['loser_rank']
+        X['elo_difference'] = (X['winner_eloRating'] - X['loser_eloRating']) * 2  # Further emphasizing ELO ratings
+
+        # Optional feature engineering based on available columns
+        if 'winner_eloRatingDelta' in X.columns and 'loser_eloRatingDelta' in X.columns:
+            X['elo_differenceDelta'] = (X['winner_eloRatingDelta'] - X['loser_eloRatingDelta']) * 2
+            X['elo_differenceDelta_abs'] = np.abs(X['elo_differenceDelta'])
+
+        # Convert dates to numeric days since the earliest date
+        try:
+            X['date'] = pd.to_datetime(X['date'], format='%Y%m%d', errors='coerce')
+            min_date = X['date'].min()
+            X['date'] = (X['date'] - min_date).dt.days
+        except Exception as e:
+            logging.warning(f"Error converting dates: {str(e)}. Setting 'date' to NaN.")
+            X['date'] = np.nan
+
+        # Convert numeric columns to float, handling errors
+        for col in self.numeric_cols:
+            X[col] = pd.to_numeric(X[col], errors='coerce')
+
+        return X
+
+def load_data(first_number=2, last_number=15):
+    """Load and combine player match data from multiple CSV files."""
+    dfs = []
+    core_columns = ['date', 'tournament', 'winner_name', 'winner_rank', 'winner_eloRating', 
+                    'loser_name', 'loser_rank', 'loser_eloRating']
+    optional_columns = ['level', 'bestOf', 'surface', 'indoor', 'speed', 'round', 
+                        'winner_seed', 'winner_country_name', 'winner_country_id', 
+                        'winner_eloRatingDelta', 'loser_seed', 'loser_country_name', 
+                        'loser_country_id', 'loser_eloRatingDelta', 'score', 'outcome', 'loser_entry']
+    all_columns = core_columns + optional_columns
+
+    dtype_dict = {
+        'loser_entry': 'object',
+        'outcome': 'object',
+        'winner_rank': 'float64',
+        'loser_rank': 'float64',
+        'winner_eloRating': 'float64',
+        'loser_eloRating': 'float64',
+        'bestOf': 'float64',
+        'speed': 'float64',
+        'winner_eloRatingDelta': 'float64',
+        'loser_eloRatingDelta': 'float64',
+        'indoor': 'float64',
+        'winner_seed': 'float64',
+        'loser_seed': 'float64'
+    }
+
+    for i in range(first_number, last_number + 1):
+        file_path = f'PlayerMatches{i}.csv'
+        try:
+            df = dd.read_csv(file_path, low_memory=False, assume_missing=True, dtype=dtype_dict)
+            missing_core_columns = [col for col in core_columns if col not in df.columns]
+            if missing_core_columns:
+                logging.warning(f"Missing core columns in {file_path}: {missing_core_columns}. Skipping this file.")
+                continue
+            available_columns = [col for col in all_columns if col in df.columns]
+            df = df[available_columns].drop_duplicates().compute()
+            dfs.append(df)
+            logging.info(f"Loaded {file_path}")
+        except FileNotFoundError:
+            logging.warning(f"{file_path} not found. Skipping this file.")
+        except Exception as e:
+            logging.warning(f"An error occurred while loading {file_path}: {str(e)}. Skipping this file.")
+
+    if not dfs:
+        raise ValueError("No valid data found.")
+
+    combined_df = pd.concat(dfs, ignore_index=True).drop_duplicates()
+    if combined_df.empty:
+        raise ValueError("The combined dataframe is empty.")
+
+    return combined_df
+
+def determine_column_types(df):
+    """Determine numeric and categorical column types in the dataframe."""
+    numeric_cols = ['winner_rank', 'loser_rank', 'winner_eloRating', 'loser_eloRating']
+    potential_numeric_cols = ['bestOf', 'speed', 'winner_eloRatingDelta', 'loser_eloRatingDelta', 'indoor', 'winner_seed', 'loser_seed']
+
+    for col in potential_numeric_cols:
+        if col in df.columns:
+            if pd.api.types.is_numeric_dtype(df[col]) or df[col].str.isnumeric().all():
+                numeric_cols.append(col)
+
+    categorical_cols = [col for col in df.columns if col not in numeric_cols and col != 'date']
+
+    return numeric_cols, categorical_cols
+
+def preprocess_data(df, numeric_cols, categorical_cols):
+    """Preprocess the dataframe, including encoding categorical variables and handling missing values."""
+    logging.info(f"Shape before preprocessing: {df.shape}")
+
+    label_encoders = {}
+    for col in categorical_cols:
+        le = LabelEncoder()
+        df[col] = df[col].astype(str)
+        df[col] = le.fit_transform(df[col])
+        label_encoders[col] = le
+
+    feature_engineer = AdvancedFeatureEngineering(numeric_cols)
+    df = feature_engineer.fit_transform(df)
+
+    logging.info(f"Shape after feature engineering: {df.shape}")
+
+    # Handle NaN values in numeric and categorical columns
+    for col in df.columns:
+        nan_count = df[col].isna().sum()
+        if nan_count > 0:
+            logging.warning(f"Column {col} has {nan_count} NaN values")
+
+    for col in numeric_cols:
+        df[col] = pd.to_numeric(df[col], errors='coerce')
+        df[col] = df[col].fillna(df[col].median())
+
+    for col in categorical_cols:
+        df[col] = df[col].fillna(-1)
+
+    # Additional check for NaNs in rank_difference
+    if df['rank_difference'].isna().any():
+        logging.error("NaN values found in 'rank_difference' after preprocessing. Identifying rows with NaN values...")
+        missing_rank_rows = df[df['rank_difference'].isna()]
+        logging.info(f"Rows with missing 'rank_difference':\n{missing_rank_rows[['winner_rank', 'loser_rank']]}")
+        df.dropna(subset=['rank_difference'], inplace=True)  # Drop rows with NaN in 'rank_difference'
+        logging.info(f"Shape after dropping NaN rows in 'rank_difference': {df.shape}")
+
+    return df, label_encoders
+
+class JointEmbeddedModel(pl.LightningModule):
+    """A PyTorch Lightning module for a neural network with categorical embeddings and numeric inputs."""
+    def __init__(self, categorical_dims, numerical_dim, embedding_dim, hidden_dim, dropout_rate=0.3, learning_rate=1e-3):
+        super().__init__()
+        self.embeddings = nn.ModuleList([nn.Embedding(dim, embedding_dim) for dim in categorical_dims])
+        self.fc1 = nn.Linear(len(categorical_dims) * embedding_dim + numerical_dim, hidden_dim)
+        self.fc2 = nn.Linear(hidden_dim, hidden_dim // 2)
+        self.fc3 = nn.Linear(hidden_dim // 2, 1)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.criterion = nn.MSELoss()
+        self.learning_rate = learning_rate
+
+    def forward(self, x_cat, x_num):
+        embedded = [emb(x_cat[:, i]) for i, emb in enumerate(self.embeddings)]
+        x = torch.cat(embedded + [x_num], dim=1)
+        x = self.dropout(self.relu(self.fc1(x)))
+        x = self.dropout(self.relu(self.fc2(x)))
+        return self.fc3(x).squeeze()
+
+    def training_step(self, batch, batch_idx):
+        x_cat, x_num, y = batch
+        y_hat = self(x_cat, x_num)
+        loss = self.criterion(y_hat, y)
+        self.log('train_loss', loss)
+        return loss
+
+    def configure_optimizers(self):
+        optimizer = optim.AdamW(self.parameters(), lr=self.learning_rate, weight_decay=1e-4)  # Using AdamW optimizer
+        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)
+        return {'optimizer': optimizer, 'lr_scheduler': scheduler, 'monitor': 'train_loss'}
+
+def create_dataloader(X, y, batch_size=64):
+    """Create a DataLoader for training and evaluation."""
+    x_cat, x_num = X
+    x_cat = torch.tensor(x_cat, dtype=torch.long)
+    x_num = torch.tensor(x_num, dtype=torch.float32)
+    y = torch.tensor(y, dtype=torch.float32)
+    dataset = TensorDataset(x_cat, x_num, y)
+    return DataLoader(dataset, batch_size=batch_size, shuffle=True)
+
+def ensemble_predictions(models, X):
+    """Aggregate predictions from an ensemble of models."""
+    preds = [model.predict(X) for model in models]
+    return np.mean(preds, axis=0)
+
+def save_model_config(config, path):
+    """Save the model configuration to a JSON file."""
+    with open(path, 'w') as f:
+        json.dump(config, f)
+
+def load_model_config(path):
+    """Load the model configuration from a JSON file."""
+    with open(path, 'r') as f:
+        return json.load(f)
+
+def objective(trial):
+    """Objective function for hyperparameter optimization with Optuna."""
+    embedding_dim = trial.suggest_int('embedding_dim', 16, 128)
+    hidden_dim = trial.suggest_int('hidden_dim', 64, 512)
+    learning_rate = trial.suggest_float('learning_rate', 1e-6, 1e-2, log=True)
+    batch_size = trial.suggest_categorical('batch_size', [32, 64, 128, 256])
+    dropout_rate = trial.suggest_float('dropout_rate', 0.1, 0.5)
+
+    model = JointEmbeddedModel(categorical_dims, numerical_dim, embedding_dim, hidden_dim, dropout_rate, learning_rate)
+    dataloader = create_dataloader(X_train, y_train, batch_size=batch_size)
+
+    trainer = pl.Trainer(
+        max_epochs=20, 
+        accelerator='gpu' if torch.cuda.is_available() else 'cpu',  # Use GPU if available
+        devices=1, 
+        logger=False,
+        enable_checkpointing=False
+    )
+    trainer.fit(model, dataloader)
+
+    val_predictions = model(torch.tensor(X_val[0], dtype=torch.long), torch.tensor(X_val[1], dtype=torch.float32)).detach().cpu().numpy()
+    if np.isnan(y_val).any() or np.isnan(val_predictions).any():
+        raise ValueError("Validation targets or predictions contain NaN values.")
+
+    val_loss = mean_squared_error(y_val, val_predictions)
+
+    return val_loss
+
+def analyze_winning_streaks(model, X, df_subset, eps=0.5, min_samples=5, threshold=0.5):
+    """Analyze winning streaks using the trained model and clustering techniques."""
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model.eval()
+    x_cat, x_num = X
+
+    with torch.no_grad():
+        embedded = [emb(torch.tensor(x_cat[:, i], dtype=torch.long).to(device)) for i, emb in enumerate(model.embeddings)]
+        embeddings = torch.cat(embedded, dim=1).cpu().numpy()
+        outputs = model(torch.tensor(x_cat, dtype=torch.long).to(device), 
+                        torch.tensor(x_num, dtype=torch.float32).to(device)).cpu().numpy()
+
+    scaler = StandardScaler()
+    embeddings = scaler.fit_transform(embeddings)
+
+    dbscan = DBSCAN(eps=eps, min_samples=min_samples)
+    labels = dbscan.fit_predict(embeddings)
+
+    df_subset['cluster'] = labels
+    df_subset['predicted_rank_difference'] = outputs
+
+    df_subset['easy_draw'] = (df_subset['rank_difference'] - df_subset['predicted_rank_difference']) > threshold
+    df_subset['hard_draw'] = (df_subset['predicted_rank_difference'] - df_subset['rank_difference']) > threshold
+
+    results = df_subset.groupby('winner_name').agg({
+        'cluster': 'count',
+        'easy_draw': 'sum',
+        'hard_draw': 'sum'
+    }).reset_index()
+
+    results['easy_draw_ratio'] = results['easy_draw'] / results['cluster']
+    results['hard_draw_ratio'] = results['hard_draw'] / results['cluster']
+
+    results.sort_values('hard_draw_ratio', ascending=False, inplace=True)
+    results.to_csv('winning_streak_analysis.csv', index=False)
+
+    logging.info(f"Analysis results saved to winning_streak_analysis.csv")
+
+    return results
+
+if __name__ == "__main__":
+    try:
+        df = load_data()
+        logging.info(f"Data loaded successfully. Shape: {df.shape}")
+
+        numeric_columns, categorical_columns = determine_column_types(df)
+        logging.info(f"Numeric columns: {numeric_columns}")
+        logging.info(f"Categorical columns: {categorical_columns}")
+
+        df, label_encoders = preprocess_data(df, numeric_columns, categorical_columns)
+        logging.info(f"Data preprocessed. Shape after preprocessing: {df.shape}")
+
+        # Ensure all numeric columns are properly handled
+        for col in numeric_columns:
+            if not pd.api.types.is_numeric_dtype(df[col]):
+                raise ValueError(f"Column {col} contains non-numeric data after preprocessing")
+
+        if df.shape[0] < N_SPLITS:
+            raise ValueError(f"Not enough samples ({df.shape[0]}) for {N_SPLITS}-fold cross-validation.")
+
+        X_cat = df[categorical_columns].values
+        X_num = df[numeric_columns].values.astype(float)
+        y = df['rank_difference'].values.astype(float)
+
+        # Remove NaN values from y
+        if np.isnan(y).any():
+            raise ValueError("Target variable contains NaN values.")
+
+        logging.info(f"Shape of X_cat: {X_cat.shape}")
+        logging.info(f"Shape of X_num: {X_num.shape}")
+        logging.info(f"Shape of y: {y.shape}")
+
+        kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)
+        scores = []
+
+        for train_index, val_index in kf.split(X_cat):
+            X_cat_train, X_cat_val = X_cat[train_index], X_cat[val_index]
+            X_num_train, X_num_val = X_num[train_index], X_num[val_index]
+            y_train, y_val = y[train_index], y[val_index]
+
+            # Additional NaN checks for validation and training sets
+            if np.isnan(X_cat_train).any() or np.isnan(X_num_train).any() or np.isnan(y_train).any():
+                raise ValueError("Training data contains NaN values.")
+            if np.isnan(X_cat_val).any() or np.isnan(X_num_val).any() or np.isnan(y_val).any():
+                raise ValueError("Validation data contains NaN values.")
+
+            X_train = (X_cat_train, X_num_train)
+            X_val = (X_cat_val, X_num_val)
+
+            categorical_dims = [len(label_encoders[col].classes_) for col in categorical_columns]
+            numerical_dim = len(numeric_columns)
+
+            try:
+                study = optuna.create_study(direction='minimize')
+                study.optimize(objective, n_trials=100)  # Further increased trials for finer parameter search
+
+                best_params = study.best_params
+                logging.info(f"Best Hyperparameters: {best_params}")
+
+                # Save the model configuration
+                model_config = {
+                    'categorical_dims': categorical_dims,
+                    'numerical_dim': numerical_dim,
+                    'embedding_dim': best_params['embedding_dim'],
+                    'hidden_dim': best_params['hidden_dim'],
+                    'dropout_rate': best_params['dropout_rate'],
+                    'learning_rate': best_params['learning_rate']
+                }
+                save_model_config(model_config, MODEL_CONFIG_PATH)
+
+                model = JointEmbeddedModel(**model_config)
+                dataloader = create_dataloader(X_train, y_train, batch_size=best_params['batch_size'])
+
+                trainer = pl.Trainer(
+                    max_epochs=100,  # Further increased max_epochs for deeper training
+                    accelerator='gpu' if torch.cuda.is_available() else 'cpu',
+                    devices=1,
+                    logger=False,
+                    enable_checkpointing=False
+                )
+                trainer.fit(model, dataloader)
+
+                val_predictions = model(torch.tensor(X_val[0], dtype=torch.long), torch.tensor(X_val[1], dtype=torch.float32)).detach().cpu().numpy()
+                if np.isnan(val_predictions).any():
+                    raise ValueError("Validation predictions contain NaN values.")
+
+                val_loss = mean_squared_error(y_val, val_predictions)
+                scores.append(val_loss)
+
+                # Save the model state
+                torch.save(model.state_dict(), BEST_MODEL_PATH)
+
+            except Exception as e:
+                logging.error(f"An error occurred during optimization: {str(e)}")
+                logging.error("Exception details:", exc_info=True)
+
+        logging.info(f"Cross-Validation MSE: {np.mean(scores):.4f}")
+
+        # Train ensemble models and evaluate
+        ensemble_models = [
+            RandomForestRegressor(n_estimators=300, random_state=RANDOM_SEED),
+            GradientBoostingRegressor(n_estimators=300, random_state=RANDOM_SEED),
+            LinearRegression()
+        ]
+
+        # Check for NaNs in ensemble training data
+        if np.isnan(np.hstack((X_cat, X_num))).any() or np.isnan(y).any():
+            raise ValueError("Ensemble training data contains NaN values.")
+
+        ensemble_models = [model.fit(np.hstack((X_cat, X_num)), y) for model in ensemble_models]
+        ensemble_preds = ensemble_predictions(ensemble_models, np.hstack((X_cat, X_num)))
+        ensemble_mse = mean_squared_error(y, ensemble_preds)
+
+        logging.info(f"Ensemble Test MSE: {ensemble_mse:.4f}")
+
+        # Load the best model configuration and state for final analysis
+        if os.path.exists(BEST_MODEL_PATH) and os.path.exists(MODEL_CONFIG_PATH):
+            model_config = load_model_config(MODEL_CONFIG_PATH)
+            model = JointEmbeddedModel(**model_config)
+            model.load_state_dict(torch.load(BEST_MODEL_PATH))
+            model.eval()
+
+            test_predictions = model(torch.tensor(X_cat, dtype=torch.long), torch.tensor(X_num, dtype=torch.float32)).detach().cpu().numpy()
+            if np.isnan(test_predictions).any():
+                raise ValueError("Test predictions contain NaN values.")
+
+            test_mse = mean_squared_error(y, test_predictions)
+            logging.info(f"Final Test MSE: {test_mse}")
+
+            winning_streak_analysis = analyze_winning_streaks(model, (X_cat, X_num), df)
+            torch.save(model.state_dict(), FINAL_MODEL_PATH)
+            logging.info("Script execution completed successfully.")
+        else:
+            logging.error("Best model or configuration not found. Ensure training is completed before running analysis.")
+
+    except Exception as e:
+        logging.error(f"An error occurred during script execution: {str(e)}")
+        logging.error("Exception details:", exc_info=True)