text_plain_output_1.png image_output_1.png
from sklearn.decomposition import PCA

import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

path = '../input/'
train = pd.read_csv(path + 'train.csv', na_values=-1)
test = pd.read_csv(path + 'test.csv', na_values=-1)

train.head(5).T

cor = train.corr()

train.isnull().sum().sum()

test.isnull().sum().sum()

train_len = train.shape[0]
df = pd.concat([train, test], axis=0)

bin_col = [c for c in df.columns if df[c].nunique() == 2]
len(bin_col)

other_col = [c for c in df.columns if c not in bin_col]
other_col

def category_type(df):
    col = df.columns
    for i in col:
        if 2 < df[i].nunique() <= 53:
            df[i] = df[i].astype('category')
category_type(df)

def OHE(df, columns):
    c2, c3 = ([], {})
    for c in columns:
        c2.append(c)
        c3[c] = 'ohe_' + c
    df1 = pd.get_dummies(df, prefix=c3, columns=c2, drop_first=True)
    return df1

col_ohe = ['X0', 'X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X8']
df1 = OHE(df, col_ohe)

pca = PCA(n_components=None, random_state=seed)
pca.fit(df1.drop(['y', 'ID'], axis=1))