text_html_output_1.png
from scipy.special import boxcox1p

import pandas as pd

train_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
test_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')

train_data.shape

test_data.shape

cat_data = train_data.select_dtypes(include='object')
cat_cols = cat_data.columns
num_data = train_data.select_dtypes(exclude='object')
num_cols = num_data.columns
num_to_cat_cols = ['MSSubClass', 'MoSold', 'YrSold', 'OverallQual', 'OverallCond']
num_cols = [i for i in num_cols if not i in num_to_cat_cols]
num_cols = [i for i in num_cols if not i in ['Id']]
num_data = num_data.drop(['Id', 'MSSubClass', 'MoSold', 'YrSold', 'OverallQual', 'OverallCond'], axis=1)

skew_dict = {}
for cols in num_cols:
    skew_dict[cols] = {'Skewness': train_data[cols].skew()}
skew_df = pd.DataFrame(skew_dict).transpose()
skew_df.columns = ['Skewness']
skew_df.sort_values(by=['Skewness'], ascending=False)

train_data = train_data.drop(train_data[train_data['Id'] == 524].index)
train_data = train_data.drop(train_data[train_data['Id'] == 1299].index)

model_data = pd.concat([train_data, test_data], axis=0, sort=None, ignore_index=True)

model_data.shape

numeric_feats = model_data.dtypes[model_data.dtypes != 'object'].index
numeric_feats = [elem for elem in numeric_feats if elem not in ('Id', 'SalePrice')]
from scipy.special import boxcox1p
lam = 0.15
for feat in numeric_feats:
    model_data[feat] = boxcox1p(model_data[feat], lam)

model_data.head()