image_output_11.png text_plain_output_5.png text_plain_output_15.png image_output_17.png text_plain_output_9.png image_output_14.png text_plain_output_4.png text_plain_output_13.png image_output_13.png image_output_5.png text_plain_output_14.png text_plain_output_10.png text_plain_output_6.png image_output_7.png text_plain_output_3.png image_output_4.png text_plain_output_7.png image_output_8.png text_plain_output_16.png image_output_16.png text_plain_output_8.png image_output_6.png image_output_12.png text_plain_output_2.png text_plain_output_1.png image_output_3.png image_output_2.png image_output_1.png image_output_10.png text_plain_output_17.png text_plain_output_11.png text_plain_output_12.png image_output_15.png image_output_9.png
import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import shutil
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
import statsmodels.api as sm
from scipy.stats import skew
from sklearn.preprocessing import OneHotEncoder
sns.set()
pd.set_option('display.max_columns', None)

pth_train = '../input/house-prices-advanced-regression-techniques/train.csv'
pth_test = '../input/house-prices-advanced-regression-techniques/test.csv'
raw_train = pd.read_csv(pth_train)
raw_test = pd.read_csv(pth_test)

categorical_nominal_cols = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating', 'Electrical', 'Functional', 'GarageType', 'MiscFeature', 'SaleType', 'SaleCondition']
categorical_ordinal_cols = ['OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageYrBlt', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
categorical_bool_cols = ['CentralAir']

categorical_ordinal2encode = {}
categorical_ordinal2encode['ExterQual'] = {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
categorical_ordinal2encode['ExterCond'] = categorical_ordinal2encode['ExterQual'].copy()
categorical_ordinal2encode['BsmtQual'] = {'NA': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
categorical_ordinal2encode['BsmtCond'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['BsmtExposure'] = {'NA': 0, 'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4}
categorical_ordinal2encode['BsmtFinType1'] = {'NA': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}
categorical_ordinal2encode['BsmtFinType2'] = categorical_ordinal2encode['BsmtFinType1'].copy()
categorical_ordinal2encode['HeatingQC'] = categorical_ordinal2encode['ExterQual'].copy()
categorical_ordinal2encode['KitchenQual'] = categorical_ordinal2encode['HeatingQC'].copy()
categorical_ordinal2encode['FireplaceQu'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['GarageFinish'] = {'NA': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
categorical_ordinal2encode['GarageQual'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['GarageCond'] = categorical_ordinal2encode['BsmtQual'].copy()
categorical_ordinal2encode['PavedDrive'] = {'N': 0, 'P': 1, 'Y': 2}
categorical_ordinal2encode['PoolQC'] = {'NA': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4}
categorical_ordinal2encode['Fence'] = {'NA': 0, 'MnWw': 1, 'GdWo': 2, 'MnPrv': 3, 'GdPrv': 4}

total_col = 4
total_row = len(categorical_ordinal_cols)//total_col
if len(categorical_ordinal_cols) % total_col > 0:
    total_row += 1
    
idx = 0
fig, axs = plt.subplots(total_row, total_col, figsize=(15,total_row * 4))
for i in range(total_row):
    for j in range(total_col):
        if idx < len(categorical_ordinal_cols): 
            title = categorical_ordinal_cols[idx]
            if title in categorical_ordinal2encode:
                vc = raw_train[title].value_counts().reset_index()
                vc.rename(columns={'index':'code'}, inplace=True)
                vc['index'] = vc['code'].copy()
                vc['index'] = vc['index'].map(categorical_ordinal2encode[title])
                vc.set_index('index', inplace=True)
                vc = vc.sort_index()
                sns.barplot(data=vc ,x='code', y=title, color='orange', ax = axs[i][j])
            else:
                vc = raw_train[title].value_counts().sort_index()
                sns.barplot(x=vc.index, y=vc, color='orange', ax = axs[i][j])
            
            axs[i][j].set_ylabel('frequency')
            axs[i][j].set_xlabel('level')

            axs[i][j].set_title(title)
            idx += 1

plt.tight_layout()
plt.show()

ordinal_columns = ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'PavedDrive', 'PoolQC', 'Fence']
ordinal_column_transforms = {}
ordinal_column_fillna = {}
ordinal_column_transforms['OverallQual'] = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [0, 0, 0, 0, 1, 2, 3, 4, 4, 4]]
ordinal_column_transforms['OverallCond'] = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [0, 0, 0, 0, 1, 2, 3, 4, 4, 4]]
ordinal_column_transforms['ExterQual'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 1]]
ordinal_column_transforms['ExterCond'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 1]]
ordinal_column_transforms['BsmtQual'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtCond'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtExposure'] = [['NA', 'No', 'Mn', 'Av', 'Gd'], [0, 0, 1, 2, 2]]
ordinal_column_transforms['BsmtFinType1'] = [['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], [0, 0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['BsmtFinType2'] = [['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], [0, 0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['HeatingQC'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 0, 1, 2, 2]]
ordinal_column_transforms['KitchenQual'] = [['Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 2, 2]]
ordinal_column_transforms['FireplaceQu'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['GarageFinish'] = [['NA', 'Unf', 'RFn', 'Fin'], [0, 1, 2, 3]]
ordinal_column_transforms['GarageQual'] = [['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], [0, 0, 1, 1, 2, 2]]
ordinal_column_transforms['PavedDrive'] = [['N', 'P', 'Y'], [0, 0, 1]]
ordinal_column_transforms['PoolQC'] = [['NA', 'Fa', 'TA', 'Gd', 'Ex'], [0, 1, 1, 2, 3]]
ordinal_column_transforms['Fence'] = [['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], [0, 0, 1, 1, 2]]

for title in ordinal_columns:
    print('raw_train',raw_train[title].isna().sum())
    print('raw_test',raw_test[title].isna().sum())
    
    vc = raw_train[title].value_counts().sort_index()
    raw_train[title].replace(ordinal_column_transforms[title][0],ordinal_column_transforms[title][1], inplace=True)
    raw_test[title].replace(ordinal_column_transforms[title][0],ordinal_column_transforms[title][1], inplace=True)
    raw_test[title].fillna(0, inplace=True)
    raw_test[title].fillna(0, inplace=True)
    
    vc_changed = raw_train[title].value_counts().sort_index()

    total_row, total_vc = raw_train.shape[0], vc.sum()
    gap = total_row - total_vc
    print('total_row :',total_row)
    print('total value count :',total_vc)
    print('total null value :',gap,'\n')
    
    fig, axs = plt.subplots(1,2,figsize=(15,3))
    if title in categorical_ordinal2encode:
        vc = vc.reset_index()
        vc.rename(columns={'index':'code'}, inplace=True)
        vc['index'] = vc['code'].copy()
        vc['index'] = vc['index'].map(categorical_ordinal2encode[title])
        vc.set_index('index', inplace=True)
        vc = vc.sort_index()
        sns.barplot(data=vc ,x='code', y=title, color='violet', ax = axs[0])
    else:
        sns.barplot(x=vc.index, y=vc, color='violet', ax=axs[0])
        axs[0].set_title('BEFORE',fontsize=12)
        axs[0].set_ylabel('frequency')
        axs[0].set_xlabel('level')

    sns.barplot(x=vc_changed.index, y=vc_changed, color='violet', ax=axs[1])
    axs[1].set_title('AFTER',fontsize=12)
    axs[1].set_ylabel('frequency')
    axs[1].set_xlabel('level')

    fig.suptitle(title+' (BEFORE - AFTER)',fontsize=15)

    # plt.title(title+' (BEFORE - AFTER)', fontsize=15)
    plt.tight_layout()
    plt.show()

raw_test['SalePrice'] = np.zeros(raw_test.shape[0], dtype=np.int64)
raw_all = pd.concat((raw_train, raw_test), axis=0)
check_null_cols = raw_all.isna().sum()
check_null_cols_ver2 = check_null_cols[check_null_cols > 0] / raw_all.shape[0] * 100
check_null_cols_ver3 = check_null_cols_ver2[check_null_cols_ver2 > 50]
raw_all_ver2 = raw_all.drop(columns=check_null_cols_ver3.index)
print('Remove feature unnecessary')
print('Before', raw_all.shape)
print('After', raw_all_ver2.shape)