text_html_output_1.png
from sklearn.preprocessing import OneHotEncoder

from sklearn.utils import resample

import pandas as pd

import seaborn as sns

train_df = train_transactions.merge(train_identity, how='left', on='TransactionID')
del train_transactions, train_identity

test_df = test_df.rename(columns=lambda x: '_'.join(x.split('-')))
train_df.set_index('TransactionID', inplace=True)
test_df.set_index('TransactionID', inplace=True)

train_df.to_pickle('train_df.pkl')
test_df.to_pickle('test_df.pkl')

object_cols = [col for col in train_df.columns if train_df[col].dtype == 'object']
good_label_cols = [col for col in object_cols if set(train_df[col]) == set(test_df[col])]
bad_label_cols = list(set(object_cols) - set(good_label_cols))

train_df.drop(bad_label_cols, axis=1, inplace=True)
test_df.drop(bad_label_cols, axis=1, inplace=True)

low_cardinality_cols = [col for col in good_label_cols if train_df[col].nunique() < 10]
high_cardinality_cols = list(set(good_label_cols) - set(low_cardinality_cols))

from sklearn.preprocessing import OneHotEncoder
OH_encoder = OneHotEncoder(drop='first', sparse=False)
OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(train_df[low_cardinality_cols].astype('str')))
OH_cols_valid = pd.DataFrame(OH_encoder.transform(test_df[low_cardinality_cols].astype('str')))
OH_cols_train.index = train_df.index
OH_cols_valid.index = test_df.index
num_X_train = train_df.drop(low_cardinality_cols, axis=1)
num_X_valid = test_df.drop(low_cardinality_cols, axis=1)
del train_df, test_df
OH_train_df = pd.concat([num_X_train, OH_cols_train], axis=1)
OH_test_df = pd.concat([num_X_valid, OH_cols_valid], axis=1)

df_majority_downsampled, y_majority_downsampled = resample(X_train[y_train == 0], y_train[y_train == 0], replace=False, n_samples=3 * len(y_train[y_train == 1]), random_state=42)
X_down_train = pd.concat([X_train[y_train == 1], df_majority_downsampled])
y_down_train = pd.concat([y_train[y_train == 1], y_majority_downsampled])
sns.countplot(x=y_down_train)