text_html_output_1.png
import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
test_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')

numerical_label = list(train_data.dtypes[train_data.dtypes != 'object'].index)
non_numerical_label = list(train_data.dtypes[train_data.dtypes == 'object'].index)
numerical_label.remove('Id')
numerical_label.remove('SalePrice')

na_per_col_train = train_data.isna().sum(axis=0)
na_per_col_test = test_data.isna().sum(axis=0)
na_per_col = np.array(na_per_col_train[0:80]) + np.array(na_per_col_test)
col2drop_train = list(train_data.columns[na_per_col_train >= train_data.shape[0] * 0.2])
col2drop_test = list(test_data.columns[na_per_col_test >= test_data.shape[0] * 0.2])
col2drop_train.extend(col2drop_test)
col2drop = np.unique(col2drop_train)
train_data = train_data.drop(col2drop, axis=1)
test_data = test_data.drop(col2drop, axis=1)
numerical_label = list(set(numerical_label).difference(set(col2drop)))
non_numerical_label = list(set(non_numerical_label).difference(set(col2drop)))

train_data[numerical_label] = train_data[numerical_label].fillna(0)
train_data[non_numerical_label] = train_data[non_numerical_label].fillna('n')
test_data[numerical_label] = test_data[numerical_label].fillna(0)
test_data[non_numerical_label] = test_data[non_numerical_label].fillna('n')
na_per_row = train_data.isna().sum(axis=1)
row2drop = list(train_data.index[na_per_row > 0])
train_data = train_data.drop(row2drop, axis=0)
print('dropped rows by SalePrice:' + ' '.join(list([str(row2drop[i]) for i in range(0, len(row2drop))])))