application_vnd.jupyter.stderr_output_9.png application_vnd.jupyter.stderr_output_7.png application_vnd.jupyter.stderr_output_11.png text_plain_output_20.png text_plain_output_4.png text_plain_output_14.png text_plain_output_10.png text_plain_output_6.png text_plain_output_21.png text_plain_output_18.png application_vnd.jupyter.stderr_output_19.png application_vnd.jupyter.stderr_output_13.png application_vnd.jupyter.stderr_output_3.png application_vnd.jupyter.stderr_output_5.png text_plain_output_16.png application_vnd.jupyter.stderr_output_15.png text_plain_output_8.png application_vnd.jupyter.stderr_output_17.png text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png text_plain_output_12.png
from sklearn import compose

from sklearn import impute

from sklearn import metrics

from sklearn import model_selection

from sklearn import pipeline

from sklearn import preprocessing

import lightgbm as lgbm

import numpy as np

import pandas as pd

import random

import xgboost as xgb

import glob
import pandas as pd
import numpy as np
import optuna
import random
from sklearn import compose
from sklearn import impute
from sklearn import linear_model
from sklearn import metrics
from sklearn import model_selection
from sklearn import pipeline
from sklearn import preprocessing
import lightgbm as lgbm
import xgboost as xgb
GPU_ENABLED = False

FOLDS = 10
df_train = pd.read_csv('../input/30-days-of-ml/train.csv')
kfold = model_selection.KFold(n_splits=FOLDS, shuffle=True, random_state=42)
df_train['fold'] = -1
for fold, (_, valid_idx) in enumerate(kfold.split(df_train)):
    df_train.loc[valid_idx, 'fold'] = fold
df_train.to_csv('folds.csv', index=False)

df_train = pd.read_csv('folds.csv')
df_test = pd.read_csv('../input/30-days-of-ml/test.csv')

cont_features = [f for f in df_train.columns.tolist() if f.startswith('cont')]
cat_features = [f for f in df_train.columns.tolist() if f.startswith('cat')]

dummies = pd.get_dummies(df_train[cat_features])
df_train[dummies.columns] = dummies.iloc[:len(df_train), :]
df_test[dummies.columns] = dummies.iloc[len(df_test):, :]

class Trainer(object):
    def __init__(self, df_train, df_test):
        self.df_train = df_train
        self.df_test = df_test
        self.predictions_valid = None
        self.predictions_test = None
    def fit(self, models):
        model_scores = dict()
        self.predictions_valid = dict()
        self.predictions_test = dict()
        for model_index in range(len(models)):
            model_scores.setdefault(model_index, [])
            self.predictions_valid.setdefault(model_index, dict())
            self.predictions_test.setdefault(model_index, [])
        self.preprocessed_data = dict()
        folds = df_train.fold.max() + 1
        for fold in range(folds):
            X_train_fold = self.df_train[self.df_train.fold != fold]
            X_valid_fold = self.df_train[self.df_train.fold == fold]
            valid_idx = X_valid_fold.id.values.tolist()
            y_train = X_train_fold.target
            y_valid = X_valid_fold.target
            for model_index, (preprocessor, features, model_fn) in enumerate(models):
                xtrain = X_train_fold[features]
                xvalid = X_valid_fold[features]
                xtest = self.df_test[features]
                preprocessor_id = f'{fold}_{id(preprocessor)}'
                if preprocessor_id not in self.preprocessed_data:
                    self.preprocessed_data[preprocessor_id] = {'X_train': preprocessor.fit_transform(xtrain, y_train), 'X_valid': preprocessor.transform(xvalid), 'X_test': preprocessor.transform(xtest)}
                X_train = self.preprocessed_data[preprocessor_id]['X_train']
                X_valid = self.preprocessed_data[preprocessor_id]['X_valid']
                X_test = self.preprocessed_data[preprocessor_id]['X_test']
                model = model_fn(fold)
                model.fit(X_train, y_train, early_stopping_rounds=100, eval_set=[(X_valid, y_valid)], verbose=False)
                yhat_valid = model.predict(X_valid)
                yhat_test = model.predict(X_test)
                self.predictions_valid[model_index].update(dict(zip(valid_idx, yhat_valid)))
                self.predictions_test[model_index].append(yhat_test)
                rmse = metrics.mean_squared_error(y_valid, yhat_valid, squared=False)
                model_scores[model_index].append(rmse)
        scores = []
        for index in range(len(models)):
            score = np.mean(model_scores[index])
            scores.append(score)
        self._save()
        return scores
    def _save(self):
        if self.predictions_valid is None:
            return
        for index in range(len(self.predictions_valid.keys())):
            model_id = random.randint(1000000000, 9999999999)
            df_predictions_valid = pd.DataFrame.from_dict(self.predictions_valid[index], orient='index').reset_index()
            df_predictions_valid.columns = ['id', f'predictions_{model_id}']
            df_predictions_valid.to_csv(f'predictions_train_model_{model_id}.csv', index=False)
            target = np.mean(np.column_stack(self.predictions_test[index]), axis=1)
            df_predictions_test = pd.DataFrame({'id': self.df_test.id, f'predictions_{model_id}': target})
            df_predictions_test.to_csv(f'predictions_test_model_{model_id}.csv', index=False)

numerical_preprocessor = pipeline.Pipeline(steps=[('imputer', impute.SimpleImputer(strategy='mean')), ('scaler', preprocessing.MinMaxScaler())])
categorical_preprocessor = pipeline.Pipeline(steps=[('imputer', impute.SimpleImputer(strategy='most_frequent')), ('ordinal', preprocessing.OrdinalEncoder())])
onehot_preprocessor = preprocessing.OneHotEncoder(handle_unknown='ignore')
preprocessor1 = compose.ColumnTransformer(transformers=[('numerical', numerical_preprocessor, cont_features), ('categorical', categorical_preprocessor, ['cat0', 'cat1', 'cat2', 'cat6', 'cat7', 'cat8', 'cat9']), ('onehot', onehot_preprocessor, ['cat3', 'cat4', 'cat5'])])
preprocessor2 = compose.ColumnTransformer(transformers=[('numerical', numerical_preprocessor, cont_features), ('categorical', categorical_preprocessor, cat_features)])
preprocessor3 = compose.ColumnTransformer(transformers=[('numerical', numerical_preprocessor, cont_features), ('categorical', categorical_preprocessor, ['cat1', 'cat5', 'cat8'])])

def model_xgb1(random_state):
    model_params = {'n_estimators': 5000, 'colsample_bytree': 0.14503672812818655, 'learning_rate': 0.026154895111343116, 'max_depth': 6, 'reg_alpha': 33.58185735675347, 'reg_lambda': 0.0006654723943414539, 'subsample': 0.7737874287937573}
    if GPU_ENABLED:
        model_params['tree_method'] = 'gpu_hist'
        model_params['predictor'] = 'gpu_predictor'
    else:
        model_params['n_jobs'] = 4
    return xgb.XGBRegressor(objective='reg:squarederror', random_state=random_state, **model_params)
def model_xgb2(random_state):
    model_params = {'n_estimators': 5000, 'colsample_bytree': 0.13404738492192508, 'learning_rate': 0.016208647588692938, 'max_depth': 6, 'reg_alpha': 17.228932163667228, 'reg_lambda': 6.33772225860826e-05, 'subsample': 0.5455951254226656}
    if GPU_ENABLED:
        model_params['tree_method'] = 'gpu_hist'
        model_params['predictor'] = 'gpu_predictor'
    else:
        model_params['n_jobs'] = 4
    return xgb.XGBRegressor(objective='reg:squarederror', random_state=random_state, **model_params)
def model_xgb3(random_state):
    model_params = {'n_estimators': 5000, 'colsample_bytree': 0.1260369083045909, 'learning_rate': 0.08194751952715394, 'max_depth': 3, 'reg_alpha': 52.63823514724582, 'reg_lambda': 3.101903064457115, 'subsample': 0.6466414975747573}
    if GPU_ENABLED:
        model_params['tree_method'] = 'gpu_hist'
        model_params['predictor'] = 'gpu_predictor'
    else:
        model_params['n_jobs'] = 4
    return xgb.XGBRegressor(objective='reg:squarederror', random_state=random_state, **model_params)
def model_lgbm(random_state):
    model_params = {'n_estimators': 20000, 'bagging_fraction': 0.8974383620638076, 'bagging_freq': 6, 'feature_fraction': 0.44911144741980047, 'lambda_l1': 8.576512805347963e-06, 'lambda_l2': 2.0513310788967636, 'learning_rate': 0.01499785501240656, 'min_child_samples': 92, 'num_leaves': 4}
    if GPU_ENABLED:
        model_params['device'] = 'gpu'
        model_params['gpu_platform_id'] = 0
        model_params['gpu_device_id'] = 0
    else:
        model_params['n_jobs'] = 4
    return lgbm.LGBMRegressor(objective='regression', metric='rmse', random_state=random_state, **model_params)

features1 = cont_features + cat_features
features2 = cont_features + ['cat1', 'cat5', 'cat8', 'cat1_A', 'cat3_C', 'cat8_C', 'cat8_E']
trainer = Trainer(df_train, df_test)
trainer.fit([(preprocessor1, features1, model_xgb1), (preprocessor2, features1, model_xgb2), (preprocessor3, features2, model_xgb3), (preprocessor2, features1, model_lgbm)])