text_plain_output_100.png text_plain_output_84.png text_plain_output_56.png text_plain_output_35.png text_plain_output_130.png text_plain_output_117.png text_plain_output_98.png text_plain_output_43.png text_plain_output_78.png text_plain_output_106.png text_plain_output_37.png text_plain_output_90.png text_plain_output_79.png text_plain_output_5.png text_plain_output_75.png text_plain_output_48.png text_plain_output_116.png text_plain_output_128.png text_plain_output_30.png text_plain_output_73.png text_plain_output_126.png text_plain_output_115.png text_plain_output_15.png text_plain_output_114.png text_plain_output_70.png text_plain_output_9.png text_plain_output_44.png text_plain_output_119.png text_plain_output_86.png text_plain_output_118.png text_plain_output_131.png text_plain_output_40.png text_plain_output_123.png text_plain_output_74.png text_plain_output_31.png text_plain_output_20.png text_plain_output_102.png text_plain_output_111.png text_plain_output_101.png text_plain_output_132.png text_plain_output_60.png text_plain_output_68.png text_plain_output_4.png text_plain_output_65.png text_plain_output_64.png text_plain_output_13.png text_plain_output_107.png text_plain_output_52.png text_plain_output_66.png text_plain_output_45.png text_plain_output_14.png text_plain_output_32.png text_plain_output_88.png text_plain_output_29.png text_plain_output_129.png text_plain_output_58.png text_plain_output_49.png text_plain_output_63.png text_plain_output_27.png text_plain_output_76.png text_plain_output_108.png text_plain_output_54.png text_plain_output_10.png text_plain_output_6.png text_plain_output_92.png text_plain_output_57.png text_plain_output_120.png text_plain_output_24.png text_plain_output_21.png text_plain_output_104.png text_plain_output_47.png text_plain_output_121.png text_plain_output_25.png text_plain_output_77.png text_plain_output_18.png text_plain_output_50.png text_plain_output_36.png text_plain_output_96.png text_plain_output_87.png text_plain_output_3.png text_plain_output_112.png text_plain_output_113.png text_plain_output_22.png text_plain_output_81.png text_plain_output_69.png text_plain_output_125.png text_plain_output_38.png text_plain_output_7.png text_plain_output_91.png text_plain_output_16.png text_plain_output_59.png text_plain_output_103.png text_plain_output_71.png text_plain_output_8.png text_plain_output_122.png text_plain_output_26.png text_plain_output_109.png text_plain_output_41.png text_plain_output_34.png text_plain_output_85.png text_plain_output_42.png text_plain_output_110.png text_plain_output_67.png text_plain_output_53.png text_plain_output_23.png text_plain_output_89.png text_plain_output_51.png text_plain_output_28.png text_plain_output_72.png text_plain_output_99.png text_plain_output_2.png text_plain_output_127.png text_plain_output_97.png text_plain_output_1.png text_plain_output_33.png text_plain_output_39.png text_plain_output_55.png text_plain_output_82.png text_plain_output_93.png text_plain_output_19.png text_plain_output_105.png text_plain_output_80.png text_plain_output_94.png text_plain_output_124.png text_plain_output_17.png text_plain_output_11.png text_plain_output_12.png text_plain_output_62.png text_plain_output_95.png text_plain_output_61.png text_plain_output_83.png text_plain_output_46.png
from collections import Counter

from keras.callbacks import History, EarlyStopping

from keras.layers import Conv1D, BatchNormalization, Dense, Flatten, Activation, Dropout

from keras.models import Sequential

from keras.utils import Sequence

from tensorflow.python.client import device_lib

from time import perf_counter

import keras

import numpy as np

import os

import pywt

import soundfile as sf

import tensorflow as tf

import os
from time import perf_counter
import numpy as np
import soundfile as sf
from collections import Counter
import matplotlib.pyplot as plt
from tensorflow.python.client import device_lib
import tensorflow as tf
import keras
from keras.layers import Conv1D, BatchNormalization, Dense, Flatten, Activation, Dropout
from tensorflow.keras.layers.experimental import preprocessing
from keras.models import Sequential
from keras.callbacks import History, EarlyStopping
history = History()
generator_init = False
model_saved = False
checkpoint_saved = False
SAVED_MODEL_PATH = './'
MODEL_NAME = SAVED_MODEL_PATH + 'langid_model'
CHECKPOINT_FILEPATH = MODEL_NAME + '_CP'
USE_OVERFITTING_ORIGINAL_NW = 1
USE_DROPOUT_NO_REGULARIZATION_NW = 2
USE_DROPOUT_REGULARIZATION_NW = 3
nn_choice = USE_DROPOUT_REGULARIZATION_NW
WAVELET = 'bior6.8'
train_path = '../input/spoken-language-identification/train/train/'
test_path = '../input/spoken-language-identification/test/test/'

from keras.utils import Sequence
import pywt
import pdb
class langidDataGenerator(Sequence):
    """Generates data for Keras"""
    def __init__(self, list_IDs, labels, wavelet='rbio3.1', drop_levels=None, batch_size=32, n_channels=1, n_classes=6, shuffle=True):
        """Initialization"""
        self.wvlt = wavelet
        self.drop_lvls = slice(0, -drop_levels)
        self.batch_size = batch_size
        self.labels = labels
        self.list_IDs = list_IDs
        self.n_channels = n_channels
        self.n_classes = n_classes
        self.shuffle = shuffle
        self.on_epoch_end()
        X, y = self.__data_generation(self.list_IDs[:2])
        X = np.expand_dims(X, 2)
        self.dim = X.shape[1:]
    def __len__(self):
        """Denotes the number of batches per epoch"""
        return int(np.floor(len(self.list_IDs) / self.batch_size))
    def __getitem__(self, index):
        """Generate one batch of data"""
        indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]
        list_IDs_temp = [self.list_IDs[k] for k in indexes]
        X, y = self.__data_generation(list_IDs_temp)
        X = tf.expand_dims(X, 2)
        return (X, y)
    def on_epoch_end(self):
        """Updates indexes after each epoch"""
        self.indexes = tf.range(len(self.list_IDs))
        if self.shuffle == True:
            tf.random.shuffle(self.indexes)
    def wavelet_features(self, list_IDs_temp):
        features = []
        y = []
        for ID in list_IDs_temp:
            signal, fs = sf.read(ID)
            list_coeff = pywt.wavedec(signal, self.wvlt, mode='per')
            dwt_local_coeff = []
            end_flag = 0
            for coeff in list_coeff[self.drop_lvls]:
                dwt_local_coeff.extend(coeff)
            features.append(dwt_local_coeff)
            y.append(self.labels[ID])
        X = tf.convert_to_tensor(features)
        return (X, y)
    def __data_generation(self, list_IDs_temp):
        """Generates data containing batch_size samples"""
        X, y = self.wavelet_features(list_IDs_temp)
        return (X, keras.utils.to_categorical(y, num_classes=self.n_classes))

"""
DATA_FOLDER = '../datasets/langid/'
train_path = DATA_FOLDER+'test/'
test_path = DATA_FOLDER+'test/'
"""
train_labels = []
for filename in os.listdir(train_path):
    train_labels.append(filename[:4])
test_labels = []
for filename in os.listdir(test_path):
    test_labels.append(filename[:4])
lb = 0
labeld = {}
for k in Counter(train_labels).keys():
    labeld[k] = lb
    lb = lb + 1
num_classes = lb
train_files = []
train_labels2 = {}
for filename in os.listdir(train_path):
    train_files.append(train_path + filename)
    train_labels2[train_path + filename] = labeld[filename[:4]]
test_files = []
test_labels2 = {}
for filename in os.listdir(test_path):
    test_files.append(test_path + filename)
    test_labels2[test_path + filename] = labeld[filename[:4]]

drop_levels = 2
"\n#ss = np.random.random_sample(2**17)\nsig_dwt = pywt.wavedec(sig,WAVELET,mode='per')\nprint('# of levels decomposed {}'.format(dec_lvls))\n"

class CustomEarlyStopping(keras.callbacks.Callback):
    def __init__(self, patience=0):
        super(CustomEarlyStopping, self).__init__()
        self.patience = patience
        self.best_weights = None
    def on_train_begin(self, logs=None):
        self.wait = 0
        self.stopped_epoch = 0
        self.best_v_loss = np.Inf
        self.best_v_accuracy = 0
    def on_epoch_end(self, epoch, logs=None):
        v_loss = logs.get('val_loss')
        v_acc = logs.get('val_accuracy')
        if np.less(v_loss, self.best_v_loss) or np.greater(v_acc, self.best_v_accuracy):
            self.best_v_loss = v_loss
            self.best_v_accuracy = v_acc
            self.wait = 0
            self.best_weights = self.model.get_weights()
        else:
            self.wait += 1
            if self.wait >= self.patience:
                self.stopped_epoch = epoch
                self.model.stop_training = True
                self.model.set_weights(self.best_weights)
    def on_train_end(self, logs=None):
        pass

def dropout_block(model, no_nodes, dropout, reg=None, activation='relu'):
    model.add(Dense(units=no_nodes, kernel_regularizer=reg))
    model.add(Dropout(dropout))
    model.add(BatchNormalization())
    model.add(Activation(activation))
epochs = 128
no_train = 8000
no_test = None
params = {'wavelet': WAVELET, 'drop_levels': drop_levels, 'batch_size': 32, 'n_classes': 6, 'n_channels': 1, 'shuffle': True}
if not generator_init:
    training_generator = langidDataGenerator(train_files[:no_train], train_labels2, **params)
    validation_generator = langidDataGenerator(test_files[:no_test], test_labels2, **params)
'\nif not data_normalized:\n    training_generator.normalize_data()\n'
if not model_saved:
    model = Sequential()
    model.add(BatchNormalization(input_shape=training_generator.dim))
    model.add(Dropout(0.2))
    model.add(Conv1D(32, kernel_size=9, strides=3))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv1D(16, kernel_size=5, strides=2))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv1D(8, kernel_size=3))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv1D(1, kernel_size=3))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Flatten())
    if nn_choice == USE_OVERFITTING_ORIGINAL_NW:
        model.add(Dense(1024, kernel_regularizer=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05)))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Dense(256, kernel_regularizer=keras.regularizers.l1_l2(l1=0.0005, l2=0.0005)))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Dense(64, kernel_regularizer=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05)))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Dense(16, kernel_regularizer=keras.regularizers.l1_l2(l1=1e-05, l2=1e-05)))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Flatten())
        model.add(Dense(num_classes, kernel_regularizer=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05), activation='softmax'))
    elif nn_choice == USE_DROPOUT_REGULARIZATION_NW:
        dropout_block(model, no_nodes=2 ** 13, dropout=0.3, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=2 ** 12, dropout=0.3, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=2048, dropout=0.3, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=1536, dropout=0.2, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=1024, dropout=0.2, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=784, dropout=0.2, reg=keras.regularizers.l1_l2(l1=0.0005, l2=0.0005))
        dropout_block(model, no_nodes=512, dropout=0.2, reg=keras.regularizers.l1_l2(l1=0.0005, l2=0.0005))
        dropout_block(model, no_nodes=256, dropout=0.15, reg=keras.regularizers.l1_l2(l1=0.0005, l2=0.0005))
        dropout_block(model, no_nodes=128, dropout=0.15, reg=keras.regularizers.l1_l2(l1=0.0005, l2=0.0005))
        dropout_block(model, no_nodes=64, dropout=0.1, reg=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05))
        dropout_block(model, no_nodes=16, dropout=0.1, reg=keras.regularizers.l1_l2(l1=1e-05, l2=1e-05))
        model.add(Flatten())
        model.add(Dense(num_classes, kernel_regularizer=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05), activation='softmax'))
    elif nn_choice == USE_DROPOUT_NO_REGULARIZATION_NW:
        dropout_block(model, no_nodes=2048, dropout=0.4)
        dropout_block(model, no_nodes=1536, dropout=0.3)
        dropout_block(model, no_nodes=1024, dropout=0.3)
        dropout_block(model, no_nodes=784, dropout=0.3)
        dropout_block(model, no_nodes=512, dropout=0.3)
        dropout_block(model, no_nodes=256, dropout=0.25)
        dropout_block(model, no_nodes=128, dropout=0.25)
        dropout_block(model, no_nodes=64, dropout=0.2)
        dropout_block(model, no_nodes=16, dropout=0.1)
        model.add(Dense(num_classes, kernel_regularizer=keras.regularizers.l1_l2(l1=5e-05, l2=5e-05), activation='softmax'))
    model.summary()
    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
else:
    model = keras.models.load_model(MODEL_NAME)
es = CustomEarlyStopping(patience=4)
cp = tf.keras.callbacks.ModelCheckpoint(filepath=CHECKPOINT_FILEPATH, save_weights_only=True, save_freq=16)
if checkpoint_saved:
    model.load_weights(CHECKPOINT_FILEPATH)
t_start = perf_counter()
model.fit_generator(generator=training_generator, validation_data=validation_generator, epochs=epochs, use_multiprocessing=False, verbose=1, callbacks=[history, es, cp])
'\nmodel.fit(training_generator, epochs=epochs, verbose=1,\n          callbacks=[history,es])\n'
t_stop = perf_counter()
t_diff = t_stop - t_start
print('Time to train the network {} seconds'.format(t_diff))
train_score = model.evaluate(training_generator, verbose=0)
print('Train loss: {}, Train accuracy: {}'.format(train_score[0], train_score[1]))
test_score = model.evaluate(validation_generator, verbose=0)
print('Test loss: {}, Test accuracy: {}'.format(test_score[0], test_score[1]))