I think there are several points to take into account:
* First, it is possible that, in this case, the default XGBoost hyperparameters are a better combination that the ones your are passing through your params__grid combinations, you could check for it
* Although it does not explain your case, keep in mind that the best_score given by the GridSearchCV object is the Mean cross-validated score of the best_estimator (source), it is, the mean test score across the k trainings over the k-folds, so it can maybe give you a more reliable score value (including also the standard deviation of the k scores)
Consider also that you do not need to retrain your model with the best params fron the grid search CV, since you can access the best model via search.best_model
Lastly, I would also recommend other hyperparameter tuning methods like the bayesian tuning, below a sample code gist using hyperopt (info about bayesian optimization here):
from hyperopt import hp
from sklearn.model_selection import StratifiedKFold
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.model_selection import cross_val_score
def return_model_scores(params, dataX, dataY, n_folds=5):
import numpy as np
defined_model, edp_cost_scores, histories =
evaluate_model_cost(dataX = dataX, dataY = dataY)
return np.array(edp_cost_scores).mean()
param_space = {'max_depth' : hp.choice('max_depth', range(5, 30, 1)),
'learning_rate' : hp.quniform('learning_rate', 0.01, 0.5, 0.01),
'n_estimators' : hp.choice('n_estimators', range(20, 205, 5)),
'gamma' : hp.quniform('gamma', 0, 0.50, 0.01),
'min_child_weight' : hp.quniform('min_child_weight', 1, 10, 1),
'subsample' : hp.quniform('subsample', 0.1, 1, 0.01),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1.0, 0.01),
'scale_pos_weight': hp.uniform('scale_pos_weight', 0.2, 0.8)}
# Some variable
dataX = X_train
dataY = y_train
assert len(X_train)==len(y_train)
n_folds=5
global best # global variable defined for convenience of this use case
best = 0
i = 0
def f(params):
cost = return_model_scores(params, dataX, dataY, n_folds=5)
if i == 0:
best = cost
if cost < best:
best = cost
print('new best:', best, params)
return {'loss': cost, 'status': STATUS_OK}
trials = Trials()
best = fmin(f, param_space, algo=tpe.suggest, max_evals=10, trials=trials)
print ('best:')
print (best)
where evaluate_model_cost is the function you build for getting the cost values after evaluation:
def evaluate_model_cost(dataX, dataY):
from tqdm import tqdm
from sklearn.model_selection import KFold
scores, histories = list(), list()
# prepare cross validation
kfold = KFold(10, shuffle=True, random_state=1)
# enumerate splits
k = 0
for train_ix, test_ix in tqdm(kfold.split(dataX)):
print('kfold {}'.format(k))
# select rows for train and test
trainX, trainY, testX, testY = dataX.iloc[train_ix],
dataY.iloc[train_ix], dataX.iloc[test_ix], dataY.iloc[test_ix]
# fit model
history = defined_model.fit(trainX, trainY,
eval_metric= 'auc',
eval_set=[(testX, testY)])
# evaluate model
y_preds = defined_model. 392