text_plain_output_1.png
"""def run(trial):
    #optimize in one fold
    fold = 0
    xtrain = train[train.kfold != fold].reset_index(drop=True)
    xvalid = train[train.kfold == fold].reset_index(drop=True)
    ytrain = xtrain.target
    yvalid = xvalid.target
    xtrain = xtrain[features]
    xvalid = xvalid[features]
    xtrain[object_cols]= ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.fit_transform(xvalid[object_cols])
    learning_rate = trial.suggest_float("learning_rate", 1e-2, 0.8, log=True)
    colsample_bytree = trial.suggest_float('colsample_bytree', 0.1, 0.6)
    max_depth = trial.suggest_int('max_depth', 1, 9)
    subsample = trial.suggest_float('subsample', 0.1, 0.6)
    reg_lambda = trial.suggest_float('reg_lambda', 1e-5, 100.0)
    reg_alpha = trial.suggest_float('reg_alpha', 1e-5, 100.0)
    alpha = trial.suggest_int('alpha', 0, 100)
    model = XGBRegressor(random_state = 0, 
                         alpha=alpha,
                         n_estimators=200, 
                         tree_method='gpu_hist',
                         gpu_id=0, predictor='gpu_predictor',
                         learning_rate = learning_rate,
                         colsample_bytree = colsample_bytree,
                         max_depth = max_depth,
                         subsample = subsample,
                         reg_lambda = reg_lambda,
                         reg_alpha = reg_alpha)
    model.fit(xtrain, ytrain)
    preds_valid = model.predict(xvalid)
    rmse = mean_squared_error(yvalid, preds_valid, squared=False)
    return rmse
study = optuna.create_study(direction="minimize")
study.optimize(run, n_trials=5000)
#study.best_params
final_preds = []
for fold in range(5):
    xtrain = train[train.kfold != fold].reset_index(drop=True)
    xvalid = train[train.kfold == fold].reset_index(drop=True)
    ytrain = xtrain.target
    yvalid = xvalid.target
    xtrain = xtrain[features]
    xvalid = xvalid[features]
    xtrain[object_cols]= ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.fit_transform(xvalid[object_cols])
    best_params = {'learning_rate': 0.34090767065203226,
                     'colsample_bytree': 0.12289350813119115,
                     'max_depth': 7,
                     'reg_lambda': 5.830490094721956,
                     'reg_alpha': 49.68136144185203,
                     'alpha': 30
                  }
    model = XGBRegressor(objective='reg:squarederror',
                         n_estimators=200,
                         random_state=0,
                         **best_params
                         #tree_method='gpu_hist',
                         #gpu_id=0, 
                         #predictor='gpu_predictor'
                        )
    model.fit(xtrain, ytrain)
    preds_valid = model.predict(xvalid)
    test_preds = model.predict(xtest)
    final_preds.append(test_preds)
    print(fold, mean_squared_error(yvalid, preds_valid, squared=False))
"""