text_plain_output_1.png
train_df = train_transactions.merge(train_identity, how='left', on='TransactionID')
del train_transactions, train_identity

test_df = test_df.rename(columns=lambda x: '_'.join(x.split('-')))
train_df.set_index('TransactionID', inplace=True)
test_df.set_index('TransactionID', inplace=True)

train_df.to_pickle('train_df.pkl')
test_df.to_pickle('test_df.pkl')

object_cols = [col for col in train_df.columns if train_df[col].dtype == 'object']
good_label_cols = [col for col in object_cols if set(train_df[col]) == set(test_df[col])]
bad_label_cols = list(set(object_cols) - set(good_label_cols))

train_df.drop(bad_label_cols, axis=1, inplace=True)
test_df.drop(bad_label_cols, axis=1, inplace=True)

low_cardinality_cols = [col for col in good_label_cols if train_df[col].nunique() < 10]
high_cardinality_cols = list(set(good_label_cols) - set(low_cardinality_cols))
print('Categorical columns that will be one-hot encoded:', low_cardinality_cols)
print('\nCategorical columns that will be label encoded:', high_cardinality_cols)