text_plain_output_1.png
from sklearn.feature_selection import mutual_info_regression

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')

(df_train.shape, df_test.shape)

numerical_features = [cols for cols in df_train if df_train[cols].dtype in ['int', 'float']]
categorical_features = [cols for cols in df_train if df_train[cols].dtype == 'object']

numerical_missing_columns = list(df_train[numerical_features].columns[df_train[numerical_features].isna().any()])

(df_train[numerical_missing_columns].isnull().sum() / df_train.shape[0] * 100).sort_values(ascending=False)

data_frames = [df_train, df_test]
for df in data_frames:
    df.drop('GarageYrBlt', axis=1, inplace=True)
numerical_features.remove('GarageYrBlt')

categorical_features_to_investigate = [col for col in categorical_features if df_train[col].nunique() <= 6]
categorical_features_to_delete = [col for col in categorical_features if col not in categorical_features_to_investigate]
data_frames = [df_train, df_test]
for df in data_frames:
    df.drop(categorical_features_to_delete, axis=1, inplace=True)

categorical_missing_columns = list(df_train[categorical_features_to_investigate].columns[df_train[categorical_features_to_investigate].isnull().any()])
(df_train[categorical_missing_columns].isnull().sum() / df_train.shape[0] * 100).sort_values(ascending=False)

df_test.isnull().sum()

ordinal_var_dict = {'Street': ['Grvl', 'Pave'], 'LotShape': ['IR3', 'IR2', 'IR1', 'Reg'], 'LandContour': ['Low', 'HLS', 'Bnk', 'Lvl'], 'LandSlope': ['Sev', 'Mod', 'Gtl'], 'ExterQual': ['Fa', 'TA', 'Gd', 'Ex'], 'CentralAir': ['N', 'Y'], 'PavedDrive': ['N', 'P', 'Y']}
for var in ordinal_var_dict:
    ordered_var = pd.api.types.CategoricalDtype(categories=ordinal_var_dict[var], ordered=True)
    df_train[var] = df_train[var].astype(ordered_var)
    df_test[var] = df_test[var].astype(ordered_var)

X = df_train.drop(['SalePrice'], axis=1)
y = df_train['SalePrice']

for colname in X.select_dtypes(include=['object', 'category']):
    X[colname], _ = X[colname].factorize()
discrete_features = X.dtypes == int
for colname in df_test.select_dtypes(include=['object', 'category']):
    df_test[colname], _ = df_test[colname].factorize()

from sklearn.feature_selection import mutual_info_regression

mi_scores = mutual_info_regression(X, y)
mi_scores = pd.Series(mi_scores, name='Mi scores', index=X.columns)
mi_scores = mi_scores.sort_values(ascending=False)

selected_mi = mi_scores[mi_scores >= 0.2].index
X = X[selected_mi]
X