text_plain_output_1.png
from functools import partial

from scipy.stats.mstats import winsorize

from sklearn.metrics import mean_squared_error

from sklearn.model_selection import StratifiedKFold

from tabular import TabularTransformer, DataGenerator

from tabular import gelu, Mish, mish

from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau

from tensorflow.keras.layers import Dense

from tensorflow.keras.layers import Flatten, concatenate, Concatenate, Lambda, Dropout, SpatialDropout1D

from tensorflow.keras.layers import Input, Embedding, Reshape, GlobalAveragePooling1D

from tensorflow.keras.layers import Reshape, MaxPooling1D,BatchNormalization, AveragePooling1D, Conv1D

from tensorflow.keras.models import Model, load_model

from tensorflow.keras.models import load_model

from tensorflow.keras.optimizers import Adam, Nadam

from tensorflow.keras.optimizers import SGD, Adam, Nadam

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import tensorflow as tf

X_train = pd.read_csv('../input/30-days-of-ml/train.csv')
X_test = pd.read_csv('../input/30-days-of-ml/test.csv')

y_train = X_train.target
X_train = X_train.set_index('id').drop('target', axis='columns')
X_test = X_test.set_index('id')

y_stratified = pd.cut(y_train.rank(method='first'), bins=10, labels=False)

categoricals = [item for item in X_train.columns if 'cat' in item]

dummies = pd.get_dummies(X_train.append(X_test)[categoricals])
X_train[dummies.columns] = dummies.iloc[:len(X_train), :]
X_test[dummies.columns] = dummies.iloc[len(X_train):, :]
del dummies

important_features = ['cat8_E', 'cont0', 'cont5', 'cont7', 'cont8', 'cat1_A', 'cont2', 'cont13', 'cont3', 'cont10', 'cont1', 'cont9', 'cont11', 'cat1', 'cat8_C', 'cont6', 'cont12', 'cat5', 'cat3_C', 'cont4', 'cat8']
X_train = X_train[important_features]
X_test = X_test[important_features]

numeric_variables = [item for item in X_train.columns if 'cont' in item]
low_card_categoricals = ['cat1', 'cat8_E', 'cat1_A', 'cat8_C', 'cat3_C']
high_card_categoricals = ['cat5', 'cat8']
ordinals = ['cat5', 'cat8']

def tabular_dnn(numeric_variables, categorical_variables=None, categorical_counts=None, feature_selection_dropout=0.2, categorical_dropout=0.1, first_dense=256, second_dense=256, dense_dropout=0.2, activation_type=gelu):
    numerical_inputs = Input(shape=(numeric_variables,))
    numerical_normalization = BatchNormalization()(numerical_inputs)
    numerical_feature_selection = Dropout(feature_selection_dropout)(numerical_normalization)
    if categorical_variables is not None:
        categorical_inputs = []
        categorical_embeddings = []
        for category in categorical_variables:
            categorical_inputs.append(Input(shape=[1], name=category))
            category_counts = categorical_counts[category]
            categorical_embeddings.append(Embedding(category_counts + 1, int(np.log1p(category_counts) + 1), name=category + '_embed')(categorical_inputs[-1]))
        if len(categorical_embeddings) == 1:
            categorical_logits = Flatten()(SpatialDropout1D(categorical_dropout)(categorical_embeddings[0]))
        else:
            categorical_logits = Concatenate(name='categorical_conc')([Flatten()(SpatialDropout1D(categorical_dropout)(cat_emb)) for cat_emb in categorical_embeddings])
        xs = concatenate([numerical_feature_selection, categorical_logits])
    else:
        xs = numerical_feature_selection
    x = Dense(first_dense, activation=activation_type)(xs)
    x = Dropout(dense_dropout)(x)
    x = Dense(second_dense, activation=activation_type)(x)
    x = concatenate([x, xs])
    x = Dropout(dense_dropout)(x)
    output = Dense(1)(x)
    if categorical_variables is not None:
        model = Model([numerical_inputs] + categorical_inputs, output)
    else:
        model = Model([numerical_inputs], output)
    return model

# Useful functions

def RMSE(y_true, y_pred):
    return tf.py_function(partial(mean_squared_error, squared=False), (y_true, y_pred), tf.double)

def MSE(y_true, y_pred):
    return tf.py_function(partial(mean_squared_error, squared=True), (y_true, y_pred), tf.double)

def compile_model(model, loss, metrics, optimizer):
    model.compile(loss=loss, metrics=metrics, optimizer=optimizer)
    return model

def plot_keras_history(history, measures):
    """
    history: Keras training history
    measures = list of names of measures
    """
    rows = len(measures) // 2 + len(measures) % 2
    fig, panels = plt.subplots(rows, 2, figsize=(15, 5))
    plt.subplots_adjust(top = 0.99, bottom=0.01, hspace=0.4, wspace=0.2)
    try:
        panels = [item for sublist in panels for item in sublist]
    except:
        pass
    for k, measure in enumerate(measures):
        panel = panels[k]
        panel.set_title(measure + ' history')
        panel.plot(history.epoch, history.history[measure], label="Train "+measure)
        panel.plot(history.epoch, history.history["val_"+measure], label="Validation "+measure)
        panel.set(xlabel='epochs', ylabel=measure)
        panel.legend()
        
    plt.show(fig)

measure_to_monitor = 'val_RMSE'
modality = 'min'
early_stopping = EarlyStopping(monitor=measure_to_monitor, mode=modality, patience=5, verbose=0)
reduce_lr_on_plateau = ReduceLROnPlateau(monitor=measure_to_monitor, mode=modality, patience=2, factor=0.5, verbose=0)
model_checkpoint = ModelCheckpoint('best.model', monitor=measure_to_monitor, mode=modality, save_best_only=True, verbose=0)

skf = StratifiedKFold(n_splits=Config.folds, shuffle=True, random_state=Config.seed)
score = list()
oof = np.zeros(len(X_train))
best_iteration = list()
test_preds = np.zeros(len(X_test))
for fold, (train_idx, test_idx) in enumerate(skf.split(X_train, y_stratified)):
    tb = TabularTransformer(numeric=numeric_variables, ordinal=ordinals, lowcat=low_card_categoricals, highcat=high_card_categoricals)
    tb.fit(X_train.iloc[train_idx])
    sizes = tb.shape(X_train.iloc[train_idx])
    categorical_levels = dict(zip(high_card_categoricals, sizes[1:]))
    print(f'Input array sizes: {sizes}')
    print(f'Categorical levels: {categorical_levels}\n')
    model = tabular_dnn(numeric_variables=sizes[0], categorical_variables=high_card_categoricals, categorical_counts=categorical_levels, feature_selection_dropout=0.0, categorical_dropout=0.0, first_dense=128, second_dense=64, dense_dropout=0.1, activation_type='relu')
    model = compile_model(model, loss='mean_squared_error', metrics=[MSE, RMSE], optimizer=Adam(learning_rate=0.001))
    train_batch = DataGenerator(X_train.iloc[train_idx], np.array(winsorize(y_train[train_idx], [0.002, 0.0])), tabular_transformer=tb, batch_size=Config.batch_size, shuffle=True)
    history = model.fit(train_batch, validation_data=(tb.transform(X_train.iloc[test_idx]), y_train[test_idx]), epochs=Config.epochs, callbacks=[model_checkpoint, reduce_lr_on_plateau, early_stopping], verbose=1)
    print('\nFOLD %i' % fold)
    plot_keras_history(history, measures=['RMSE', 'MSE'])
    best_iteration.append(np.argmin(history.history['val_RMSE']) + 1)
    model = load_model('best.model', custom_objects={'gelu': gelu, 'mish': mish, 'MSE': MSE, 'RMSE': RMSE})
    preds = model.predict(tb.transform(X_train.iloc[test_idx]), verbose=1, batch_size=1024).flatten()
    oof[test_idx] = preds
    score.append(mean_squared_error(y_true=y_train[test_idx], y_pred=preds, squared=False))
    test_preds += model.predict(tb.transform(X_test[X.columns]), verbose=1, batch_size=1024).flatten()
test_preds /= Config.folds