text_plain_output_2.png text_plain_output_1.png
from sklearn.decomposition import PCA

import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

train_data = pd.read_csv('../input/tabular-playground-series-mar-2021/train.csv')
test_data = pd.read_csv('../input/tabular-playground-series-mar-2021/test.csv')

all_data = pd.concat([train_data, test_data])
all_data

train_data.shape

all_data.drop('id', axis=1, inplace=True)
train_data.drop('id', axis=1, inplace=True)
test_data.drop('id', axis=1, inplace=True)

train_data.isnull().sum()

corr_matrix = train_data.corr()

numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
num_columns = [col for col in train_data.columns if train_data[col].dtype in numerics and col != 'target']
num_columns

cat_columns = [col for col in train_data.columns if train_data[col].dtype not in numerics]
cat_columns

#thanks to @ANDRESHG

num_rows, num_cols = len(num_columns),2
f, axes = plt.subplots(nrows=num_rows, ncols=num_cols, figsize=(15, 15))
f.suptitle('Distribution of Features', fontsize=16)

for index, column in enumerate(num_columns):
    i,j = (index // num_cols, index % num_cols)
    sns.kdeplot(train_data.loc[train_data['target'] == 0, column], color="r", shade=True, ax=axes[index,0])
    sns.kdeplot(train_data.loc[train_data['target'] == 1, column], color="g", shade=True, ax=axes[index,0])
    sns.histplot(data=train_data,x=column,hue='target', kde=False, palette='Paired_r', bins=10, ax=axes[index,1],multiple='stack')

pca = PCA(n_components=1)
cont1_2 = pca.fit_transform(all_data[['cont1', 'cont2']])
cont1_2

all_data['cont1_2'] = cont1_2
all_data.drop('cont1', axis=1, inplace=True)
all_data.drop('cont2', axis=1, inplace=True)

pca1 = PCA(n_components=1)
cont0_10 = pca1.fit_transform(all_data[['cont0', 'cont10']])
cont0_10

all_data['cont0_10'] = cont0_10
all_data.drop('cont10', axis=1, inplace=True)
all_data.drop('cont0', axis=1, inplace=True)

all_data = all_data[['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9', 'cat10', 'cat11', 'cat12', 'cat13', 'cat14', 'cat15', 'cat16', 'cat17', 'cat18', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont1_2', 'cont0_10', 'target']]

train_data = all_data.iloc[0:300000, :]
test_data = all_data.iloc[300000:, :].drop(['target'], axis=1)

corr_matrix_new = train_data.corr()

corr_matrix_new[corr_matrix_new > 0.8][corr_matrix != 1].fillna('OK')