text_plain_output_1.png
from keras.preprocessing.image import ImageDataGenerator

from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV

from tensorflow import keras

from tensorflow.keras import layers

import pandas as pd

import tensorflow as tf

import tensorflow_addons as tfa

learning_rate = 0.001
weight_decay = 0.0001
batch_size = 256
num_epochs = 1000
image_size = 72
patch_size = 6
num_patches = (image_size // patch_size) ** 2
projection_dim = 64
num_heads = 4
transformer_units = [projection_dim * 2, projection_dim]
transformer_layers = 8
mlp_head_units = [2048, 1024]

input_shape = (72, 72, 3)
num_classes = 7
image_dir = '/kaggle/input/balanced-datasets/Smote_dataset'
df = pd.read_csv('/kaggle/input/balanced-datasets/Smote_dataset/labels.csv')
train_val_df, test_df = train_test_split(df, stratify=df['label'], test_size=0.1, random_state=42)
train_df, validation_df = train_test_split(train_val_df, stratify=train_val_df['label'], test_size=0.2, random_state=42)
augmented_datagen = ImageDataGenerator(rescale=1.0 / 255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode='nearest')
datagen = ImageDataGenerator(rescale=1.0 / 255)
train_generator = augmented_datagen.flow_from_dataframe(train_df, directory=image_dir, batch_size=batch_size, target_size=(image_size, image_size), x_col='filename', y_col='label', class_mode='categorical', shuffle=True)
val_generator = datagen.flow_from_dataframe(validation_df, directory=image_dir, batch_size=batch_size, target_size=(image_size, image_size), x_col='filename', y_col='label', class_mode='categorical', shuffle=True)
test_generator = datagen.flow_from_dataframe(test_df, directory=image_dir, x_col='filename', y_col='label', target_size=(image_size, image_size), batch_size=20, class_mode='categorical')

def mlp(x, hidden_units, dropout_rate):
    for units in hidden_units:
        x = layers.Dense(units, activation=tf.nn.gelu)(x)
        x = layers.Dropout(dropout_rate)(x)
    return x

class Patches(layers.Layer):
    def __init__(self, patch_size):
        super().__init__()
        self.patch_size = patch_size
    def call(self, images):
        batch_size = tf.shape(images)[0]
        patches = tf.image.extract_patches(images=images, sizes=[1, self.patch_size, self.patch_size, 1], strides=[1, self.patch_size, self.patch_size, 1], rates=[1, 1, 1, 1], padding='VALID')
        patch_dims = patches.shape[-1]
        patches = tf.reshape(patches, [batch_size, -1, patch_dims])
        return patches

class PatchEncoder(layers.Layer):
    def __init__(self, num_patches, projection_dim):
        super().__init__()
        self.num_patches = num_patches
        self.projection = layers.Dense(units=projection_dim)
        self.position_embedding = layers.Embedding(input_dim=num_patches, output_dim=projection_dim)
    def call(self, patch):
        positions = tf.range(start=0, limit=self.num_patches, delta=1)
        encoded = self.projection(patch) + self.position_embedding(positions)
        return encoded

def create_vit_classifier():
    inputs = layers.Input(shape=input_shape)
    patches = Patches(patch_size)(inputs)
    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)
    for _ in range(transformer_layers):
        x1 = layers.LayerNormalization(epsilon=1e-06)(encoded_patches)
        attention_output = layers.MultiHeadAttention(num_heads=num_heads, key_dim=projection_dim, dropout=0.1)(x1, x1)
        x2 = layers.Add()([attention_output, encoded_patches])
        x3 = layers.LayerNormalization(epsilon=1e-06)(x2)
        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)
        encoded_patches = layers.Add()([x3, x2])
    representation = layers.LayerNormalization(epsilon=1e-06)(encoded_patches)
    representation = layers.Flatten()(representation)
    representation = layers.Dropout(0.5)(representation)
    features = mlp(representation, hidden_units=mlp_head_units, dropout_rate=0.5)
    logits = layers.Dense(num_classes)(features)
    model = keras.Model(inputs=inputs, outputs=logits)
    return model

def run_experiment(model):
    optimizer = tfa.optimizers.AdamW(learning_rate=learning_rate, weight_decay=weight_decay)
    model.compile(optimizer=optimizer, loss=keras.losses.CategoricalCrossentropy(from_logits=True), metrics=[keras.metrics.CategoricalAccuracy(name='accuracy'), keras.metrics.TopKCategoricalAccuracy(3, name='top-3-accuracy')])
    mc = keras.callbacks.ModelCheckpoint(filepath='/tmp/checkpoint', monitor='val_accuracy', save_best_only=True, save_weights_only=True)
    cb = [mc]
    history = model.fit(train_generator, steps_per_epoch=len(train_df) // batch_size, epochs=num_epochs, validation_data=val_generator, validation_steps=len(validation_df) // batch_size, callbacks=cb)
    model.load_weights('/tmp/checkpoint')
    _, accuracy, top_5_accuracy = model.evaluate(test_generator)
    print(f'Test accuracy: {round(accuracy * 100, 2)}%')
    print(f'Test top 5 accuracy: {round(top_5_accuracy * 100, 2)}%')
    return history
vit_classifier = create_vit_classifier()
history = run_experiment(vit_classifier)