text_plain_output_1.png
from sklearn.metrics import mean_squared_error

from sklearn.model_selection import KFold

import catboost

import numpy as np

import pandas as pd

import time

def submit(model, test_features, test_ids, filename):
    loss_pred = model.predict(test_features)
    submission = pd.DataFrame({'id': test_ids, 'loss': loss_pred.reshape(-1)})
    submission.to_csv(filename, index=False)

train = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2021/train.csv')
test = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2021/test.csv')

train.shape

corr_score = train.corr()

train.pop('id')
test_ids = test.pop('id')

train_mean = train.mean()
train_std = train.std()

train_targets_mean = train_mean.pop('loss')
train_targets_std = train_std.pop('loss')

train_targets, validation_targets = (train_features.pop('loss'), validation_features.pop('loss'))

should_scale = True
if should_scale == True:
    train_features = (train_features - train_mean) / train_std
    validation_features = (validation_features - train_mean) / train_std
    test_features = (test - train_mean) / train_std

import catboost
import time
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
begin = time.time()
parameters = {'depth': [6, 7, 8], 'learning_rate': [0.08, 0.1], 'iterations': [300, 350]}
def train_catboost(hyperparameters, X_train, X_val, y_train, y_val):
    keys = hyperparameters.keys()
    best_index = {key: 0 for key in keys}
    best_cat = None
    best_score = 1000000000.0
    for index, key in enumerate(keys):
        items = hyperparameters[key]
        best_parameter = None
        temp_best = 1000000000.0
        for key_index, item in enumerate(items):
            iterations = hyperparameters['iterations'][best_index['iterations']] if key != 'iterations' else item
            learning_rate = hyperparameters['learning_rate'][best_index['learning_rate']] if key != 'learning_rate' else item
            depth = hyperparameters['depth'][best_index['depth']] if key != 'depth' else item
            cat = catboost.CatBoostRegressor(iterations=iterations, learning_rate=learning_rate, depth=depth)
            cat.fit(X_train, y_train, verbose=False)
            y_pred = cat.predict(X_val)
            score = np.sqrt(mean_squared_error(y_val, y_pred))
            if score < temp_best:
                temp_best = score
                best_index[key] = key_index
                best_parameter = item
            if score < best_score:
                best_score = score
                best_cat = cat
    best_parameters = {'iterations': hyperparameters['iterations'][best_index['iterations']], 'learning_rate': hyperparameters['learning_rate'][best_index['learning_rate']], 'depth': hyperparameters['depth'][best_index['depth']]}
    return (best_cat, best_score, best_parameters)
best_cat, best_score, best_parameters = train_catboost(parameters, train_features, validation_features, train_targets, validation_targets)
elapsed = time.time() - begin
submit(best_cat, test_features, test_ids, 'submission.csv')

from sklearn.model_selection import KFold
fold = 1
for train_indices, val_indices in KFold(n_splits=5, shuffle=True).split(train):
    print('Training with Fold %d' % fold)
    X_train = train.iloc[train_indices]
    X_val = train.iloc[val_indices]
    y_train = X_train.pop('loss')
    y_val = X_val.pop('loss')
    X_train = (X_train - train_mean) / train_std
    X_val = (X_val - train_mean) / train_std
    cat = catboost.CatBoostRegressor(iterations=best_parameters['iterations'], learning_rate=best_parameters['learning_rate'], depth=best_parameters['depth'])
    cat.fit(X_train, y_train, verbose=False)
    y_pred = cat.predict(X_val)
    score = np.sqrt(mean_squared_error(y_val, y_pred))
    print('RMSE: %.2f' % score)
    submit(cat, test_features, test_ids, 'submission_fold%d.csv' % fold)
    fold += 1