text_plain_output_1.png
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')

(df_train.shape, df_test.shape)

numerical_features = [cols for cols in df_train if df_train[cols].dtype in ['int', 'float']]
categorical_features = [cols for cols in df_train if df_train[cols].dtype == 'object']

numerical_missing_columns = list(df_train[numerical_features].columns[df_train[numerical_features].isna().any()])

(df_train[numerical_missing_columns].isnull().sum() / df_train.shape[0] * 100).sort_values(ascending=False)

data_frames = [df_train, df_test]
for df in data_frames:
    df.drop('GarageYrBlt', axis=1, inplace=True)
numerical_features.remove('GarageYrBlt')

categorical_features_to_investigate = [col for col in categorical_features if df_train[col].nunique() <= 6]
categorical_features_to_delete = [col for col in categorical_features if col not in categorical_features_to_investigate]
data_frames = [df_train, df_test]
for df in data_frames:
    df.drop(categorical_features_to_delete, axis=1, inplace=True)

categorical_missing_columns = list(df_train[categorical_features_to_investigate].columns[df_train[categorical_features_to_investigate].isnull().any()])
(df_train[categorical_missing_columns].isnull().sum() / df_train.shape[0] * 100).sort_values(ascending=False)

list_of_suspicious = ['ExterQual', 'ExterCond', 'HeatingQC', 'KitchenQual']
for col in list_of_suspicious:
    print(f'{col} \n{df_train[col].unique()}')
    print(f'{col} \n{df_test[col].unique()}')
    print('\n')