image_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
test_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')

numerical_label = list(train_data.dtypes[train_data.dtypes != 'object'].index)
non_numerical_label = list(train_data.dtypes[train_data.dtypes == 'object'].index)
numerical_label.remove('Id')
numerical_label.remove('SalePrice')

na_per_col_train = train_data.isna().sum(axis=0)
na_per_col_test = test_data.isna().sum(axis=0)
na_per_col = np.array(na_per_col_train[0:80]) + np.array(na_per_col_test)
col2drop_train = list(train_data.columns[na_per_col_train >= train_data.shape[0] * 0.2])
col2drop_test = list(test_data.columns[na_per_col_test >= test_data.shape[0] * 0.2])
col2drop_train.extend(col2drop_test)
col2drop = np.unique(col2drop_train)
train_data = train_data.drop(col2drop, axis=1)
test_data = test_data.drop(col2drop, axis=1)
numerical_label = list(set(numerical_label).difference(set(col2drop)))
non_numerical_label = list(set(non_numerical_label).difference(set(col2drop)))

train_data[numerical_label] = train_data[numerical_label].fillna(0)
train_data[non_numerical_label] = train_data[non_numerical_label].fillna('n')
test_data[numerical_label] = test_data[numerical_label].fillna(0)
test_data[non_numerical_label] = test_data[non_numerical_label].fillna('n')
na_per_row = train_data.isna().sum(axis=1)
row2drop = list(train_data.index[na_per_row > 0])
train_data = train_data.drop(row2drop, axis=0)

dup_rows_train = train_data.duplicated()
dup_rows_train = test_data.duplicated()

train_data.shape

plt.rcParams['figure.figsize'] = [25, 25]

panels=int(np.ceil(np.sqrt(len(numerical_label))))
fig, axs = plt.subplots(panels, panels)

selected_numical_features=[]

for i in range(0,len(numerical_label)):
    x=list(train_data[numerical_label[i]])
    y=list(train_data['SalePrice'])
    
    r=np.corrcoef(x, y)
    r=r[0,1]
    
    axs[i//panels,i%panels].scatter(x,y)
    axs[i//panels,i%panels].title.set_text(numerical_label[i]+": "+str(r))
    
    if r>0.5 or r<-0.5:
        selected_numical_features.append(numerical_label[i])

selected_numical_features