text_plain_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

import warnings

warnings.simplefilter('ignore')
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

train_transaction = pd.read_csv('../input/ieee-fraud-detection/train_transaction.csv')
train_identity = pd.read_csv('../input/ieee-fraud-detection/train_identity.csv')
test_transaction = pd.read_csv('../input/ieee-fraud-detection/test_transaction.csv')
test_identity = pd.read_csv('../input/ieee-fraud-detection/test_identity.csv')
sample_submission = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')

train_df = train_transaction.merge(train_identity, how='left', on='TransactionID')
test_df = test_transaction.merge(test_identity, how='left', on='TransactionID')

test_df = test_df.rename(columns=lambda x: '_'.join(x.split('-')))

def high_correlated_cols(dataframe, plot=True, corr_th=0.85):
    corr = dataframe.corr()
    cor_matrix = corr.abs()
    upper_triangle_matrix = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(np.bool))
    drop_list = [col for col in upper_triangle_matrix.columns if any(upper_triangle_matrix[col] > corr_th)]
    if plot:
        sns.set(rc={'figure.figsize': (25, 25)})
    return drop_list

cols = ['V' + str(x) for x in range(1, 26)]
high_correlated_cols(train_df[cols], plot=True)
drop_list1 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(26, 51)]
high_correlated_cols(train_df[cols], plot=True)
drop_list2 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(51, 76)]
high_correlated_cols(train_df[cols], plot=True)
drop_list3 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(76, 101)]
high_correlated_cols(train_df[cols], plot=True)
drop_list4 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(101, 126)]
high_correlated_cols(train_df[cols], plot=True)
drop_list5 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(126, 151)]
high_correlated_cols(train_df[cols], plot=True)
drop_list6 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(151, 176)]
high_correlated_cols(train_df[cols], plot=True)
drop_list7 = high_correlated_cols(train_df[cols], plot=False)

cols = ['V' + str(x) for x in range(176, 201)]
high_correlated_cols(train_df[cols], plot=True)
drop_list8 = high_correlated_cols(train_df[cols], plot=False)