text_plain_output_1.png
from colorama import Fore, Back, Style

from keras.utils import to_categorical

from pathlib import Path

from sklearn.decomposition import PCA

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

from sklearn.metrics import log_loss

from sklearn.model_selection import StratifiedKFold, KFold

from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler, QuantileTransformer, RobustScaler

from sklearn.preprocessing import OrdinalEncoder

from tensorflow import keras

from tensorflow.keras import layers, models, Sequential

from umap import UMAP

import numpy as np

import pandas as pd

import tensorflow as tf

import tensorflow_ranking as tfr

INPUT_DIR = Path('/kaggle/input/playground-series-s3e13/')
train_data = pd.read_csv(INPUT_DIR / 'train.csv')
train_data['data_type'] = 0
train_data['prognosis'] = train_data['prognosis'].str.replace(' ', '_')
test_data = pd.read_csv(INPUT_DIR / 'test.csv')
test_data['data_type'] = 0
features = sorted(list(set(test_data.columns) - set(['id', 'data_type'])))
if INCLUDE_ORIGINAL:
    df_original = pd.read_csv('/kaggle/input/vector-borne-disease-prediction/trainn.csv')
    df_original['id'] = -1 - np.arange(len(df_original))
    df_original['prognosis'] = df_original['prognosis'].str.replace(' ', '_')
    df_original['data_type'] = 1
    train_data = pd.concat([df_original, train_data]).reset_index(drop=True)
    features = sorted(list(set(test_data.columns) - set(['id'])))

from sklearn.preprocessing import OrdinalEncoder
from keras.utils import to_categorical
enc = OrdinalEncoder()
y_enc = enc.fit_transform(train_data.filter(['prognosis']))
y = to_categorical(y_enc)
prognosis_classes = list(enc.categories_[0])
N_CLASSES = len(prognosis_classes)

feats = list(features)
X = train_data.filter(feats).values
X_train = train_data.drop('data_type', axis=1).filter(feats).values
X_data_type = train_data['data_type'].values
X_test = test_data.drop('data_type', axis=1).filter(feats).values
X_tst = test_data.filter(feats).values

encoder_dim = 12
dim_input = layers.Input(shape=(64,))
encoded_layer_0 = tf.keras.layers.GaussianNoise(0.1)(dim_input)
encoded_layer_1 = layers.Dense(60, activation='relu')(encoded_layer_0)
encoded_layer_2 = layers.Dense(50, activation='relu')(encoded_layer_1)
encoded_layer_3 = layers.Dense(40, activation='relu')(encoded_layer_2)
encoded_layer_4 = layers.Dense(30, activation='relu')(encoded_layer_3)
encoded_layer_5 = layers.Dense(20, activation='relu')(encoded_layer_4)
encoded_layer_6 = layers.Dense(encoder_dim, activation='softmax')(encoded_layer_5)
decoded_layer_1 = layers.Dense(20, activation='relu')(encoded_layer_6)
decoded_layer_2 = layers.Dense(30, activation='relu')(decoded_layer_1)
decoded_layer_3 = layers.Dense(40, activation='relu')(decoded_layer_2)
decoded_layer_4 = layers.Dense(50, activation='relu')(decoded_layer_3)
decoded_layer_5 = layers.Dense(60, activation='relu')(decoded_layer_4)
decoded_layer_6 = layers.Dense(64, activation='relu')(decoded_layer_5)
autoencoder = keras.Model(inputs=dim_input, outputs=decoded_layer_6)
autoencoder.compile(loss='categorical_crossentropy', metrics='categorical_accuracy')

encoder = keras.Model(inputs=dim_input, outputs=encoded_layer_6)
encoded_input = layers.Input(shape=(encoder_dim,))

encoded_train = pd.DataFrame(encoder.predict(X_train))
encoded_train = encoded_train.add_prefix('feature_')
encoded_test = pd.DataFrame(encoder.predict(X_test))
encoded_test = encoded_test.add_prefix('feature_')

class Decomp:
    def __init__(self, n_components, method='pca', scaler_method='standard'):
        self.n_components = n_components
        self.method = method
        self.scaler_method = scaler_method
    def dimension_reduction(self, df, y=None):
        if self.method == 'LDA':
            X_reduced = self.dimension_method(df, y)
        else:
            X_reduced = self.dimension_method(df)
        df_comp = pd.DataFrame(X_reduced, columns=[f'{self.method.upper()}_{_}' for _ in range(self.n_components)])
        return df_comp
    def dimension_method(self, df, y=None):
        X = self.scaler(df)
        if self.method == 'pca':
            comp = PCA(n_components=self.n_components, random_state=0)
            X_reduced = comp.fit_transform(X)
        elif self.method == 'nmf':
            comp = NMF(n_components=self.n_components, random_state=0)
            X_reduced = comp.fit_transform(X)
        elif self.method == 'umap':
            comp = UMAP(n_components=self.n_components, random_state=0)
            X_reduced = comp.fit_transform(X)
        elif self.method == 'tsne':
            comp = TSNE(n_components=self.n_components, random_state=0)
            X_reduced = comp.fit_transform(X)
        elif self.method == 'LDA':
            comp = LinearDiscriminantAnalysis(n_components=self.n_components)
            X_reduced = comp.fit_transform(X, y)
        else:
            raise ValueError(f'Invalid method name: {method}')
        self.comp = comp
        return X_reduced
    def scaler(self, df):
        _df = df.copy()
        if self.scaler_method == 'standard':
            return StandardScaler().fit_transform(_df)
        elif self.scaler_method == 'minmax':
            return MinMaxScaler().fit_transform(_df)
        elif self.scaler_method == None:
            return _df
        else:
            raise ValueError(f'Invalid scaler_method name')
    def get_columns(self):
        return [f'{self.method.upper()}_{_}' for _ in range(self.n_components)]
    def transform(self, df, y=None):
        X = self.scaler(df)
        X_reduced = self.comp.transform(X)
        df_comp = pd.DataFrame(X_reduced, columns=[f'{self.method.upper()}_{_}' for _ in range(self.n_components)])
        return df_comp
    @property
    def get_explained_variance_ratio(self):
        return np.sum(self.comp.explained_variance_ratio_)

n_components = 5
decomp = Decomp(n_components=n_components, method='umap', scaler_method=None)
umap_train = decomp.dimension_reduction(X_train)
umap_test = decomp.transform(X_test)

n_components = 7
decomp = Decomp(n_components=n_components, method='pca', scaler_method=None)
pca_train = decomp.dimension_reduction(X_train)
pca_test = decomp.transform(X_test)

n_components = 8
decomp = Decomp(n_components=n_components, method='LDA', scaler_method=None)
lda_train = decomp.dimension_reduction(X_train, y_enc.flatten())
lda_test = decomp.transform(X_test)

new_all_data = pd.concat([train_data, umap_train, lda_train], axis=1)
new_test = pd.concat([test_data, umap_test, lda_test], axis=1)

def build_model(input_shape):
    def mish(x):
        return keras.layers.Lambda(lambda x: x * K.tanh(K.softplus(x)))(x)
    inputs = layers.Input(shape=input_shape)
    x = layers.Dropout(0.2)(inputs)
    x = layers.Dense(64, activation=mish)(x)
    for _ in range(1):
        x = layers.BatchNormalization()(x)
        x = layers.Dense(32, activation=mish)(x)
        x = layers.Dropout(0.5)(x)
    outputs = layers.Dense(N_CLASSES, activation='softmax')(x)
    return keras.Model(inputs, outputs)
epochs = 100
lr_start = 0.002
lr_end = 1e-07
def cosine_decay(epoch):
    if epochs > 1:
        w = (1 + np.cos(epoch / (epochs - 1) * np.pi)) / 2
    else:
        w = 1
    return w * lr_start + (1 - w) * lr_end
lr = keras.callbacks.LearningRateScheduler(cosine_decay, verbose=0)
callbacks = [lr, keras.callbacks.TerminateOnNaN()]

def map3_from_logloss(y_enc, preds, data_type):
    return map3(y_enc[data_type == 0], preds[data_type == 0])
def fold_logloss(y_enc, preds, data_type):
    return log_loss(y_enc[data_type == 0], preds[data_type == 0])

N_FOLDS = 10
N_REPEATS = 10
features = sorted(list(set(new_test.columns) - set(['id', 'data_type'])))
feats = list(features)
X = new_all_data.filter(feats).values.astype(np.float32)
X_data_type = new_all_data['data_type'].values.astype(np.float32)
X_test = new_test.filter(feats).values.astype(np.float32)
oof_preds = np.zeros((len(train_data), N_CLASSES))
test_preds = np.zeros((len(test_data), N_CLASSES))
oof_metrics = []
oof_lls = []
for i in range(N_REPEATS):
    k_fold = StratifiedKFold(n_splits=N_FOLDS, random_state=RANDOM_STATE + i, shuffle=True)
    for train_index, test_index in k_fold.split(X, y_enc.flatten()):
        X_train, X_valid = (X[train_index], X[test_index])
        y_train, y_valid = (y[train_index], y[test_index])
        model = build_model(input_shape=(len(feats),))
        model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss='categorical_crossentropy', metrics=[tfr.keras.metrics.MeanAveragePrecisionMetric(topn=3)])
        history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid), batch_size=32, epochs=epochs, callbacks=callbacks, verbose=0)
        oof_pred = model.predict(X_valid, verbose=0)
        test_pred = model.predict(X_test, verbose=0)
        oof_metric = map3_from_logloss(y_valid, oof_pred, X_data_type[test_index])
        oof_ll = fold_logloss(y_valid, oof_pred, X_data_type[test_index])
        oof_metrics.append(oof_metric)
        oof_lls.append(oof_ll)
        oof_preds[test_index] += oof_pred / N_REPEATS
        test_preds += test_pred / (N_REPEATS * N_FOLDS)
oof_metric = np.round(np.mean(oof_metrics), 5)
oof_ll = np.round(np.mean(oof_lls), 5)
print(f'{Fore.GREEN}{Style.BRIGHT}Average metric = {round(oof_metric, 5)}{Style.RESET_ALL}')