text_plain_output_1.png
import pandas as pd

train_df_sample = pd.read_csv('../input/amex-default-prediction/train_data.csv', nrows=100000)

train_label_df = pd.read_csv('../input/amex-default-prediction/train_labels.csv')

test_df = pd.read_csv('../input/amex-default-prediction/test_data.csv', nrows=100000, index_col='customer_ID')

train_df = pd.merge(train_df_sample, train_label_df, how='inner', on=['customer_ID'])

train_df.drop(axis=1, columns=['customer_ID', 'S_2'], inplace=True)

test_df.drop(axis=1, columns=['S_2'], inplace=True)

i = 0
for col in train_df.columns:
    if train_df[col].isnull().sum() / len(train_df[col]) * 100 >= 75:
        train_df.drop(labels=col, axis=1, inplace=True)
        i = i + 1

i = 0
for col in test_df.columns:
    if test_df[col].isnull().sum() / len(test_df[col]) * 100 >= 75:
        print('Dropping column', col)
        test_df.drop(labels=col, axis=1, inplace=True)
        i = i + 1
print('Total number of columns dropped in test dataframe', i)