text_plain_output_1.png
from sklearn.metrics import mean_absolute_error, roc_auc_score

from sklearn.model_selection import train_test_split, cross_val_predict, cross_val_score, StratifiedKFold, RandomizedSearchCV

from sklearn.preprocessing import RobustScaler

from xgboost import XGBRegressor, XGBClassifier

import matplotlib.pyplot as plt

import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

data = pd.read_csv('/kaggle/input/playground-series-s3e7/train.csv')

corrmat = data.corr()
cols = corrmat.nlargest(20, 'booking_status')['booking_status'].index
cm = np.corrcoef(data[cols].values.T)
fig = plt.gcf()
fig.set_size_inches(10, 8)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)

g = sns.PairGrid(data, y_vars=['booking_status'], x_vars=data.columns)
g = g.map_diag(plt.hist)
g = g.map_offdiag(sns.scatterplot)

y = data.booking_status
X = data.drop(['booking_status', 'id'], axis=1)
X_train, X_val, y_train, y_val = train_test_split(X, y, train_size=0.8, test_size=0.2)

scaler = RobustScaler()
X = pd.DataFrame(scaler.fit_transform(X))
X_train = pd.DataFrame(scaler.fit_transform(X_train))
X_val = pd.DataFrame(scaler.fit_transform(X_val))

param_grid = {'n_estimators': [1750, 2000, 2250, 2500], 'learning_rate': [0.05, 0.065, 0.08, 0.9], 'max_depth': [3, 4, 5, 6, 8, 10, 12, 15], 'min_child_weight': [3, 4, 5, 6, 7], 'gamma': [0.2, 0.3, 0.4, 0.5, 0.6], 'colsample_bytree': [0.4, 0.5, 0.6, 0.65, 0.7]}

RS_XGB = RandomizedSearchCV(estimator=XGBClassifier(early_stopping_rounds=30), param_distributions=param_grid, n_iter=3, cv=6, n_jobs=-1, random_state=0)
RS_XGB.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_val, y_val)], verbose=0)

RS_XGB.best_params_

xgb_models = []
scores = []
feature_importances = []
skf = StratifiedKFold(n_splits=30, shuffle=True, random_state=0)
for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
    X_train, y_train = (X.iloc[train_idx], y.iloc[train_idx])
    X_valid, y_valid = (X.iloc[val_idx], y.iloc[val_idx])
    evaluation = [(X_train, y_train), (X_valid, y_valid)]
    model = XGBClassifier(**RS_XGB.best_params_, n_jobs=-1, early_stopping_rounds=300)
    model.fit(X_train, y_train, eval_set=evaluation, verbose=0)
    val_preds = model.predict_proba(X_valid)[:, 1]
    val_score = roc_auc_score(y_valid, val_preds)
    best_iter = model.best_iteration
    feature_importances.append({i: j for i in model.feature_names_in_ for j in model.feature_importances_})
    print(f' auc :{val_score:.5f}    best iteration  :{best_iter}')
    if val_score > 0.9:
        scores.append(val_score)
        xgb_models.append(model)
mean_val_auc = np.array(scores).mean(0)
print(f'Mean AUC:  {mean_val_auc}')