text_plain_output_1.png image_output_1.png
from sklearn.model_selection import KFold, train_test_split, StratifiedKFold

from tensorflow import keras

from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array

from tqdm.keras import TqdmCallback

import numpy as np

import pandas as pd

train = pd.read_csv('../input/digit-recognizer/train.csv')
test = pd.read_csv('../input/digit-recognizer/test.csv')
sample = pd.read_csv('../input/digit-recognizer/sample_submission.csv')

y = train.pop('label')

plt.rcParams['figure.figsize'] = (12, 18)
_ = plt.imshow(array_to_img(train.values.reshape(-1, 28, 28, 1)[10]), cmap = 'gray')
plt.axis('off')
print(y[10])

kf = StratifiedKFold(n_splits=10, shuffle=True, random_state=4)

def build_model():
    inputs = keras.layers.Input(shape=(28, 28, 1))
    conv1 = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(inputs)
    conv2 = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(conv1)
    maxpool1 = keras.layers.MaxPool2D((2, 2))(conv2)
    conv3 = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(maxpool1)
    conv4 = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(conv3)
    maxpool2 = keras.layers.MaxPool2D((2, 2))(conv4)
    flat = keras.layers.Flatten()(maxpool2)
    x1 = keras.layers.Dense(1024, activation='relu')(flat)
    x1 = keras.layers.Dropout(0.1)(x1)
    x = keras.layers.Concatenate(axis=1)([flat, x1])
    x2 = keras.layers.Dense(512, activation='relu')(x)
    x2 = keras.layers.Dropout(0.1)(x2)
    x = keras.layers.Concatenate(axis=1)([flat, x1, x2])
    x3 = keras.layers.Dense(256, activation='relu')(x)
    x3 = keras.layers.Dropout(0.1)(x3)
    x = keras.layers.Concatenate(axis=1)([flat, x1, x2, x3])
    x4 = keras.layers.Dense(128, activation='relu')(x)
    x4 = keras.layers.Dropout(0.1)(x4)
    x = keras.layers.Concatenate(axis=1)([flat, x1, x2, x3, x4])
    x = keras.layers.Dense(64, activation='relu')(x)
    x = keras.layers.Dense(10, activation='softmax')(x)
    model = keras.models.Model(inputs=inputs, outputs=x)
    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=keras.losses.SparseCategoricalCrossentropy(), metrics=['accuracy'])
    return model

def build_model2():
    inputs = keras.layers.Input(shape=(28, 28, 1))
    conv = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(inputs)
    conv = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(conv)
    maxpool = keras.layers.MaxPool2D((2, 2))(conv)
    conv = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(maxpool)
    conv = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(conv)
    maxpool = keras.layers.MaxPool2D((2, 2))(conv)
    flat = keras.layers.Flatten()(maxpool)
    x1 = keras.layers.Dense(1024, activation='relu')(flat)
    x1 = keras.layers.Dropout(0.1)(x1)
    x2 = keras.layers.Dense(512, activation='relu')(x1)
    x2 = keras.layers.Dropout(0.1)(x2)
    x3 = keras.layers.Dense(256, activation='relu')(x2)
    x3 = keras.layers.Dropout(0.1)(x3)
    x4 = keras.layers.Dense(128, activation='relu')(x3)
    x4 = keras.layers.Dropout(0.1)(x4)
    x1_ = keras.layers.Dense(1024, activation='relu')(flat)
    x1_ = keras.layers.Dropout(0.1)(x1_)
    x2_ = keras.layers.Dense(512, activation='relu')(x1_)
    x2_ = keras.layers.Dropout(0.1)(x2_)
    x3_ = keras.layers.Dense(256, activation='relu')(x2_)
    x3_ = keras.layers.Dropout(0.1)(x3_)
    x4_ = keras.layers.Dense(128, activation='relu')(x3_)
    x4_ = keras.layers.Dropout(0.1)(x4_)
    x = keras.layers.Concatenate(axis=1)([flat, x4, x4_])
    x = keras.layers.Dense(64, activation='relu')(x)
    x = keras.layers.Dense(10, activation='softmax')(x)
    model = keras.models.Model(inputs=inputs, outputs=x)
    model.compile(optimizer='adam', loss=keras.losses.SparseCategoricalCrossentropy(), metrics=['accuracy'])
    return model

def build_model3():
    inputs = keras.layers.Input(shape=(28, 28, 1))
    conv = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(inputs)
    conv = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(conv)
    maxpool = keras.layers.MaxPool2D((2, 2))(conv)
    conv = keras.layers.Conv2D(64, (3, 3), strides=1, activation='relu')(maxpool)
    conv = keras.layers.Conv2D(32, (3, 3), strides=1, activation='relu')(conv)
    maxpool = keras.layers.MaxPool2D((2, 2))(conv)
    flat = keras.layers.Flatten()(maxpool)
    x1 = keras.layers.Dense(1024, activation='relu')(flat)
    x1 = keras.layers.Dropout(0.1)(x1)
    x2 = keras.layers.Dense(512, activation='relu')(x1)
    x2 = keras.layers.Dropout(0.1)(x2)
    x3 = keras.layers.Dense(256, activation='relu')(x2)
    x3 = keras.layers.Dropout(0.1)(x3)
    x4 = keras.layers.Dense(128, activation='relu')(x3)
    x4 = keras.layers.Dropout(0.1)(x4)
    x1_ = keras.layers.Dense(1024, activation='relu')(flat)
    x1_ = keras.layers.Dropout(0.1)(x1_)
    x2_ = keras.layers.Dense(512, activation='relu')(x1_)
    x2_ = keras.layers.Dropout(0.1)(x2_)
    x3_ = keras.layers.Dense(256, activation='relu')(x2_)
    x3_ = keras.layers.Dropout(0.1)(x3_)
    x4_ = keras.layers.Dense(128, activation='relu')(x3_)
    x4_ = keras.layers.Dropout(0.1)(x4_)
    x = keras.layers.Concatenate(axis=1)([x1, x2, x3, x4, x1_, x2_, x3_, x4_, flat])
    x = keras.layers.Dense(64, activation='relu')(x)
    x = keras.layers.Dense(10, activation='softmax')(x)
    model = keras.models.Model(inputs=inputs, outputs=x)
    model.compile(optimizer='adam', loss=keras.losses.SparseCategoricalCrossentropy(), metrics=['accuracy'])
    return model

scores = []
test_preds = []
for i, (t, v) in enumerate(kf.split(train, y)):
    K.clear_session()
    xtrain = train.iloc[t, :].values.reshape(-1, 28, 28, 1)
    xval = train.iloc[v, :].values.reshape(-1, 28, 28, 1)
    xtest = test.copy().values.reshape(-1, 28, 28, 1)
    ytrain = y[t]
    yval = y[v]
    train_gen = ImageDataGenerator(rescale=1.0 / 255, rotation_range=20, width_shift_range=0.3, height_shift_range=0.3)
    val_gen = ImageDataGenerator(rescale=1.0 / 255)
    train_datagen = train_gen.flow(xtrain, ytrain, batch_size=64, seed=4)
    val_datagen = val_gen.flow(xval, yval, batch_size=64, seed=4)
    test_datagen = val_gen.flow(xtest, batch_size=64, shuffle=False)
    callbacks = [keras.callbacks.EarlyStopping(patience=28, mode='min', monitor='val_loss', restore_best_weights=True), keras.callbacks.ReduceLROnPlateau(patience=5, mode='min', monitor='val_loss', factor=0.5, min_lr=1e-05), TqdmCallback(verbose=1)]
    model = build_model()
    history = model.fit(train_datagen, validation_data=val_datagen, epochs=150, callbacks=callbacks, verbose=0)
    history = pd.DataFrame(history.history)
    _ = history.loc[4:, ['val_loss', 'loss']].plot()
    plt.show()
    loss, accu = model.evaluate(val_datagen)
    ypred = model.predict(test_datagen)
    test_preds.append(ypred)
    scores.append(accu)
    print(f'{i} : {accu}')
    del xtrain
    del xval
    del xtest
    del ytrain
    del yval
    del train_gen
    del val_gen
    del train_datagen
    del val_datagen
    del test_datagen
    del model
    del history
print(np.mean(scores), np.std(scores))