text_plain_output_1.png
from sklearn.preprocessing import Imputer

from subprocess import check_output

import IPython

import matplotlib #collection of functions for scientific and publication-ready visualization

import numpy as np #foundational package for scientific computing

import pandas as pd #collection of functions for data processing and analysis modeled after R dataframes with SQL like features

import scipy as sp #collection of functions for scientific computing and advance mathematics

import sklearn #collection of machine learning algorithms

import sys #access to system parameters https://docs.python.org/3/library/sys.html

import warnings

import sys
import pandas as pd
import matplotlib
import numpy as np
import scipy as sp
import IPython
from IPython import display
import sklearn
import timeit
import random
import time
import warnings
warnings.filterwarnings('ignore')
from subprocess import check_output

data_train = pd.read_csv('../input/train.csv')
data_test = pd.read_csv('../input/test.csv')

X = data_train.copy(deep=True)
X_test = data_test.copy(deep=True)

X.drop(['SalePrice'], axis=1, inplace=True)

columns_too_less = ['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscVal', 'MiscFeature']
X.drop(columns_too_less, axis=1, inplace=True)
X_test.drop(columns_too_less, axis=1, inplace=True)

columns_drop_std = X.std()[X.std() < 3].index
X.drop(columns_drop_std, axis=1, inplace=True)
X_test.drop(columns_drop_std, axis=1, inplace=True)

X = X.select_dtypes(exclude=['object'])
X_test = X_test.select_dtypes(exclude=['object'])

columns_drop_corr = X.corrwith(data_train['SalePrice'])[X.corrwith(data_train['SalePrice']) < 0.2].index

X.drop(columns_drop_corr, axis=1, inplace=True)
X_test.drop(columns_drop_corr, axis=1, inplace=True)

columns = X.columns

from sklearn.preprocessing import Imputer
imputer = Imputer()
X_numerical = imputer.fit_transform(X)
X_test_numerical = imputer.fit_transform(X_test)

X_numerical = pd.DataFrame(X_numerical, columns=columns)
X_test_numerical = pd.DataFrame(X_test_numerical, columns=columns)

print('X_numerical.shape: {}'.format(X_numerical.shape))
print('X_test_numerical.shape: {}'.format(X_test_numerical.shape))