text_plain_output_1.png
from sklearn.impute import SimpleImputer

from sklearn.model_selection import cross_val_score

from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder

from xgboost import XGBRegressor

import pandas as pd

import seaborn as sns

import pandas as pd
import seaborn as sns
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
import numpy as np
from matplotlib import pyplot as plt
data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
data.dropna(subset=['SalePrice'], inplace=True)
y = data.SalePrice
X = data.drop('SalePrice', axis=1)
test_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
X_test = test_data.drop('Id', axis=1)
test_id = test_data['Id']

def proc_data_ordinal(X, test_X=None):
    numerical_data = X[X.select_dtypes(exclude=['object']).columns]
    categorical_data = X[X.select_dtypes('object').columns]
    if test_X is not None:
        t_numerical_data = test_X[test_X.select_dtypes(exclude=['object']).columns]
        t_categorical_data = test_X[test_X.select_dtypes('object').columns]
    imputer = SimpleImputer(strategy='constant')
    imputed_num = pd.DataFrame(imputer.fit_transform(numerical_data))
    imputed_num.columns = numerical_data.columns
    if test_X is not None:
        t_imputed_num = pd.DataFrame(imputer.transform(t_numerical_data))
        t_imputed_num.columns = t_numerical_data.columns
    imputer = SimpleImputer(strategy='most_frequent')
    imputed_cat = pd.DataFrame(imputer.fit_transform(categorical_data))
    imputed_cat.columns = categorical_data.columns
    if test_X is not None:
        t_imputed_cat = pd.DataFrame(imputer.transform(t_categorical_data))
        t_imputed_cat.columns = t_categorical_data.columns
    ordinal_encoder = OrdinalEncoder()
    ord_X_train = imputed_cat.copy()
    ord_X_train[imputed_cat.columns] = ordinal_encoder.fit_transform(imputed_cat)
    if test_X is not None:
        t_ord_X_train = t_imputed_cat.copy()
        t_ord_X_train[t_imputed_cat.columns] = ordinal_encoder.transform(t_imputed_cat)
    processed_X = pd.concat([imputed_num, ord_X_train], axis=1)
    t_processed_X = None
    if test_X is not None:
        t_processed_X = pd.concat([t_imputed_num, t_ord_X_train], axis=1)
    return (processed_X, t_processed_X)

xgb_params = dict(max_depth=6, learning_rate=0.01, n_estimators=5000, min_child_weight=1, colsample_bytree=0.7, subsample=0.7, reg_alpha=0.5, reg_lambda=1.0, num_parallel_tree=1)

baseline_X, _ = proc_data_ordinal(X, X_test)
baseline_model = XGBRegressor(**xgb_params, random_state=1722)
previous_scores = -1 * cross_val_score(baseline_model, baseline_X, y, cv=5, scoring='neg_mean_absolute_error')
previous_scores.mean()

neighborhood_averages = data.groupby('Neighborhood')['SalePrice'].transform('mean')
neighborhood_map = pd.DataFrame({'Neighborhood': data['Neighborhood'], 'avg': neighborhood_averages})
data['Neighborhood'] = neighborhood_averages

neighborhood_map.groupby(['Neighborhood', 'avg']).size()
nbd_dict = {'Blmngtn': 194870.882353, 'Blueste': 137500.0, 'BrDale': 104493.75, 'BrkSide': 124834.051724, 'ClearCr': 212565.428571, 'CollgCr': 197965.773333, 'Crawfor': 210624.72549, 'Edwards': 128219.7, 'Gilbert': 192854.506329, 'IDOTRR': 100123.783784, 'MeadowV': 98576.470588, 'Mitchel': 156270.122449, 'NAmes': 145847.08, 'NPkVill': 142694.444444, 'NWAmes': 189050.068493, 'NoRidge': 335295.317073, 'NridgHt': 316270.623377, 'OldTown': 128225.300885, 'SWISU': 142591.36, 'Sawyer': 136793.135135, 'SawyerW': 186555.79661, 'Somerst': 225379.837209, 'StoneBr': 310499.0, 'Timber': 242247.447368, 'Veenker': 238772.727273}

y = data.SalePrice
X = data.drop('SalePrice', axis=1)
new_X, _ = proc_data_ordinal(X, None)
new_model = XGBRegressor(**xgb_params, random_state=1722)
previous_scores = -1 * cross_val_score(new_model, new_X, y, cv=5, scoring='neg_mean_absolute_error')
previous_scores.mean()

test_data['Neighborhood'] = test_data.Neighborhood.map(lambda n: nbd_dict[n])
y = data.SalePrice
X = data.drop('SalePrice', axis=1)
new_X, test_X = proc_data_ordinal(X, test_data.drop('Id', axis=1))

new_model = XGBRegressor(**xgb_params, random_state=1722)
new_model.fit(new_X, y)