text_html_output_1.png
from pathlib import Path

from sklearn.metrics import mean_squared_error

from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, KFold

from sklearn.preprocessing import StandardScaler

from xgboost import XGBRegressor

import pandas as pd

INPUT = Path('../input/tabular-playground-series-aug-2021')
train = pd.read_csv(INPUT / 'train.csv')
test = pd.read_csv(INPUT / 'test.csv')

train.shape

train.isnull().any().sum()

test.isnull().any().sum()

ss_features = [col for col in test.columns if 'f' in col]
ss = StandardScaler()
train[ss_features] = ss.fit_transform(train[ss_features])
test[ss_features] = ss.transform(test[ss_features])

X = train

kf = KFold(n_splits=10, shuffle=True, random_state=42)
for fold, (train_indices, valid_indices) in enumerate(kf.split(X=X)):
    X.loc[valid_indices, 'kfold'] = fold

useful_features = [col for col in X.columns if col not in ('loss', 'kfold')]

xgb_params = {'learning_rate': 0.013222817649672616, 'n_estimators': 12462, 'max_depth': 5, 'tree_method': 'gpu_hist', 'predictor': 'gpu_predictor'}

xgb = XGBRegressor(**xgb_params)

xgb_predictions = []
for fold in range(10):
    xtrain = X[X.kfold != fold].reset_index(drop=True)
    xvalid = X[X.kfold == fold].reset_index(drop=True)
    ytrain = xtrain.loss
    yvalid = xvalid.loss
    xtrain = xtrain[useful_features]
    xvalid = xvalid[useful_features]
    model = xgb
    model.fit(xtrain, ytrain, early_stopping_rounds=10, eval_set=[(xvalid, yvalid)], verbose=1000)
    preds_valid = model.predict(xvalid)
    test_preds = model.predict(test)
    xgb_predictions.append(test_preds)
    print(f'fold: {fold}, rmse: {mean_squared_error(yvalid, preds_valid, squared=False)}')