text_plain_output_2.png text_plain_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import shutil
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
import statsmodels.api as sm
from scipy.stats import skew
from sklearn.preprocessing import OneHotEncoder
sns.set()
pd.set_option('display.max_columns', None)

pth_train = '../input/house-prices-advanced-regression-techniques/train.csv'
pth_test = '../input/house-prices-advanced-regression-techniques/test.csv'
raw_train = pd.read_csv(pth_train)
raw_test = pd.read_csv(pth_test)

categorical_nominal_cols = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating', 'Electrical', 'Functional', 'GarageType', 'MiscFeature', 'SaleType', 'SaleCondition']
categorical_ordinal_cols = ['OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageYrBlt', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
categorical_bool_cols = ['CentralAir']

categorical_ordinal2encode = {}
categorical_ordinal2encode['ExterQual'] = {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
categorical_ordinal2encode['ExterCond'] = categorical_ordinal2encode['ExterQual'].copy()
categorical_ordinal2encode['BsmtQual'] = {'NA': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
categorical_ordinal2encode['BsmtCond'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['BsmtExposure'] = {'NA': 0, 'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4}
categorical_ordinal2encode['BsmtFinType1'] = {'NA': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}
categorical_ordinal2encode['BsmtFinType2'] = categorical_ordinal2encode['BsmtFinType1'].copy()
categorical_ordinal2encode['HeatingQC'] = categorical_ordinal2encode['ExterQual'].copy()
categorical_ordinal2encode['KitchenQual'] = categorical_ordinal2encode['HeatingQC'].copy()
categorical_ordinal2encode['FireplaceQu'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['GarageFinish'] = {'NA': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
categorical_ordinal2encode['GarageQual'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['GarageCond'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['PavedDrive'] = {'N': 0, 'P': 1, 'Y': 2}
categorical_ordinal2encode['PoolQC'] = {'NA': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
categorical_ordinal2encode['Fence'] = {'NA': 0, 'MnWw': 1, 'GdWo': 2, 'MnPrv': 3, 'GdPrv': 4}

total_col = 4
total_row = len(categorical_ordinal_cols)//total_col
if len(categorical_ordinal_cols) % total_col > 0:
    total_row += 1
    
idx = 0
fig, axs = plt.subplots(total_row, total_col, figsize=(15,total_row * 4))
for i in range(total_row):
    for j in range(total_col):
        if idx < len(categorical_ordinal_cols): 
            title = categorical_ordinal_cols[idx]
            if title in categorical_ordinal2encode:
                vc = raw_train[title].value_counts().reset_index()
                vc.rename(columns={'index':'code'}, inplace=True)
                vc['index'] = vc['code'].copy()
                vc['index'] = vc['index'].map(categorical_ordinal2encode[title])
                vc.set_index('index', inplace=True)
                vc = vc.sort_index()
                sns.barplot(data=vc ,x='code', y=title, color='orange', ax = axs[i][j])
            else:
                vc = raw_train[title].value_counts().sort_index()
                sns.barplot(x=vc.index, y=vc, color='orange', ax = axs[i][j])
            
            axs[i][j].set_ylabel('frequency')
            axs[i][j].set_xlabel('level')

            axs[i][j].set_title(title)
            idx += 1

plt.tight_layout()
plt.show()

ordinal_columns = ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'PavedDrive', 'PoolQC', 'Fence']
ordinal_column_transforms = {}
ordinal_column_fillna = {}
ordinal_column_transforms['OverallQual'] = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [0, 0, 0, 0, 1, 2, 3, 4, 4, 4]]
ordinal_column_transforms['OverallCond'] = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [0, 0, 0, 0, 1, 2, 3, 4, 4, 4]]
ordinal_column_transforms['ExterQual'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 1]]
ordinal_column_transforms['ExterCond'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 1]]
ordinal_column_transforms['BsmtQual'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtCond'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtExposure'] = [['NA', 'No', 'Mn', 'Av', 'Gd'], [0, 0, 1, 2, 2]]
ordinal_column_transforms['BsmtFinType1'] = [['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], [0, 0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtFinType2'] = [['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], [0, 0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['HeatingQC'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 2, 2]]
ordinal_column_transforms['KitchenQual'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 2, 2]]
ordinal_column_transforms['FireplaceQu'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['GarageFinish'] = [['NA', 'Unf', 'RFn', 'Fin'], [0, 1, 2, 3]]
ordinal_column_transforms['GarageQual'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['PavedDrive'] = [['N', 'P', 'Y'], [0, 0, 1]]
ordinal_column_transforms['PoolQC'] = [['NA', 'Fa', 'TA', 'Gd', 'Ex'], [0, 1, 1, 2, 3]]
ordinal_column_transforms['Fence'] = [['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], [0, 0, 1, 1, 2]]

for title in ordinal_columns:
    print('raw_train',raw_train[title].isna().sum())
    print('raw_test',raw_test[title].isna().sum())
    
    vc = raw_train[title].value_counts().sort_index()
    raw_train[title].replace(ordinal_column_transforms[title][0],ordinal_column_transforms[title][1], inplace=True)
    raw_test[title].replace(ordinal_column_transforms[title][0],ordinal_column_transforms[title][1], inplace=True)
    raw_test[title].fillna(0, inplace=True)
    raw_test[title].fillna(0, inplace=True)
    
    vc_changed = raw_train[title].value_counts().sort_index()

    total_row, total_vc = raw_train.shape[0], vc.sum()
    gap = total_row - total_vc
    print('total_row :',total_row)
    print('total value count :',total_vc)
    print('total null value :',gap,'\n')
    
    fig, axs = plt.subplots(1,2,figsize=(15,3))
    if title in categorical_ordinal2encode:
        vc = vc.reset_index()
        vc.rename(columns={'index':'code'}, inplace=True)
        vc['index'] = vc['code'].copy()
        vc['index'] = vc['index'].map(categorical_ordinal2encode[title])
        vc.set_index('index', inplace=True)
        vc = vc.sort_index()
        sns.barplot(data=vc ,x='code', y=title, color='violet', ax = axs[0])
    else:
        sns.barplot(x=vc.index, y=vc, color='violet', ax=axs[0])
        axs[0].set_title('BEFORE',fontsize=12)
        axs[0].set_ylabel('frequency')
        axs[0].set_xlabel('level')

    sns.barplot(x=vc_changed.index, y=vc_changed, color='violet', ax=axs[1])
    axs[1].set_title('AFTER',fontsize=12)
    axs[1].set_ylabel('frequency')
    axs[1].set_xlabel('level')

    fig.suptitle(title+' (BEFORE - AFTER)',fontsize=15)

    # plt.title(title+' (BEFORE - AFTER)', fontsize=15)
    plt.tight_layout()
    plt.show()

raw_test['SalePrice'] = np.zeros(raw_test.shape[0], dtype=np.int64)
raw_all = pd.concat((raw_train, raw_test), axis=0)
check_null_cols = raw_all.isna().sum()
check_null_cols_ver2 = check_null_cols[check_null_cols > 0] / raw_all.shape[0] * 100
check_null_cols_ver3 = check_null_cols_ver2[check_null_cols_ver2 > 50]
raw_all_ver2 = raw_all.drop(columns=check_null_cols_ver3.index)

check_null_cols = raw_all_ver2.isna().sum()
check_null_cols_ver2 = check_null_cols[check_null_cols > 0] / raw_all_ver2.shape[0] * 100
round(check_null_cols_ver2, 2)

raw_all_ver3 = raw_all_ver2.copy()
for column in check_null_cols_ver2.index:
    if column in categorical_ordinal_cols:
        vc = raw_all_ver3[column].value_counts()
        raw_all_ver3[column].fillna(vc.idxmax(), inplace=True)
    elif column in categorical_nominal_cols:
        raw_all_ver3[column].fillna('unknown', inplace=True)
    else:
        mean = raw_all_ver3[column].mean()
        raw_all_ver3[column].fillna(mean, inplace=True)

print('Check empty cell')
check_null_cols = raw_all_ver3.isna().sum()
check_null_cols_ver2 = check_null_cols[check_null_cols > 0] / raw_all_ver3.shape[0] * 100
round(check_null_cols_ver2, 2)