image_output_1.png
import matplotlib.pyplot as plt

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

train = pd.read_csv('../input/train.csv')
holdout = pd.read_csv('../input/test.csv')

correlation_matrix = train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(correlation_matrix, vmax=.8, square=True);

numeric = [feature for feature in train.columns if train.dtypes[feature] != 'object']
numeric.remove('Id')
numreicMostCorr = ['LotFrontage', 'OverallQual', 'YearBuilt', 'YearRemodAdd', '1stFlrSF', '2ndFlrSF', 'TotalBsmtSF', 'GrLivArea', 'FullBath', 'TotRmsAbvGrd', 'GarageArea', 'GarageCars']
for feature in numreicMostCorr:
    featureDF = pd.concat([train['SalePrice'], train[feature]], axis=1)

pairs = [('GarageArea', 'GarageCars'), ('YearBuilt', 'YearRemodAdd'), ('TotalBsmtSF', 'TotRmsAbvGrd'), ('GrLivArea', 'FullBath'), ('TotalBsmtSF', '1stFlrSF'), ('GrLivArea', '2ndFlrSF')]
for pair in pairs:
    featureDF = pd.concat([train[pair[0]], train[pair[1]]], axis=1)

categorical = [feature for feature in train.columns if train.dtypes[feature] == 'object']
for category in categorical:
    data = pd.concat([train[category], train['SalePrice']], axis=1)
    data[category] = data[category].astype('category')
    if data[category].isnull().any():
        data[category] = data[category].cat.add_categories(['MISSING'])
        data[category] = data[category].fillna('MISSING')
    cat_data = pd.concat([data['SalePrice'], data[category]], axis=1)
    f, ax = plt.subplots(figsize=(8, 6))
    fig = sns.boxplot(x=category, y="SalePrice", data=cat_data)
    fig.axis(ymin=0, ymax=800000)
    plt.show()

numeric_missing = train[numreicMostCorr].isnull().sum().sort_values(ascending=False)
categorical_missing = train[categorical].isnull().sum().sort_values(ascending=False)

numeric_to_delete = numeric_missing[numeric_missing > 438].index
categorical_to_delete = categorical_missing[categorical_missing > 438].index
def removeFromList(sourceList, filterList):
    filteredList = list(filter(lambda x: x not in filterList, sourceList))
    return filteredList
numreicMostCorr = removeFromList(numreicMostCorr, numeric_to_delete)
categorical = removeFromList(categorical, categorical_to_delete)
numreicMostCorr = removeFromList(numreicMostCorr, ['GarageCars', '1stFlrSF', '2ndFlrSF', 'YearRemodAdd', 'FullBath'])
categorical = removeFromList(categorical, ['Alley', 'LotShape', 'LandSlope', 'BldgType', 'Exterior1st', 'Exterior2nd', 'ExterCond', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'GarageFinish', 'GarageType', 'GarageCond', 'Fence'])

all_columns = numreicMostCorr + categorical + ['SalePrice']
train = train[all_columns]
train['LotFrontage'] = train['LotFrontage'].fillna(train['LotFrontage'].mean())
train[numreicMostCorr].isnull().sum().sort_values(ascending=False)
train['BsmtQual'] = train['BsmtQual'].fillna('Missing')
train['GarageQual'] = train['GarageQual'].fillna('Missing')
train['MasVnrType'] = train['MasVnrType'].fillna('Missing')
train = train.drop(train.loc[train['Electrical'].isnull()].index)
train[categorical].isnull().sum().sort_values(ascending=False)