<repo_name>Oguzhanka/face_attractiveness<file_sep>/helper.py
"""
This script is inspired from the work of:
        https://danijar.com/structuring-your-tensorflow-models/
        Author: <NAME>

Lazy property decorator is used in the model of my project.
"""
import functools
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()


def cache_func(func):
    attr = '_cache_' + func.__name__

    @property
    @functools.wraps(func)
    def decorator(self):
        if not hasattr(self, attr):
            setattr(self, attr, func(self))
        return getattr(self, attr)

    return decorator


INIT_METHODS = {"xavier": tf.initializers.glorot_normal,
                "gaussian": tf.random_normal_initializer,
                "uniform": tf.uniform_unit_scaling_initializer}
<file_sep>/config.py
"""
Configuration file to change the model & experiment parameters.
"""

IMAGE_SIZE = (80, 80, 3)                    # Spatial dimensions of the image.


class DataParams:
    """
    Configuration Parameters related to data preprocessing.
    """
    def __init__(self):
        self.input_size = IMAGE_SIZE[0]
        self.input_dims = IMAGE_SIZE[2]


class ModelParams:
    """
    Model parameters.
    """
    def __init__(self):
        """
        :attr model_type: Name of the model type. Choose large or compact.
        :attr batch_size: Batch size for the images.
        :attr num_epochs: Number of epochs.
        :attr weight_init: Weight initialization method. Choose xavier or gaussian.
        :attr learning_rate: Learning rate.
        :attr batch_norm: Flag toggling the batch normalization layers.
        :attr loss_type: Loss type for the training. Choos l1 or l2.
        :attr l2_loss: Flag toggling the L2 weight regularization.
        :attr alpha: L2 regularization parameter.
        :attr dropout: Flag toggling the dropout layers.
        :attr keep_rate: Dropout keep probability.
        """
        self.model_type = "compact"
        self.batch_size = 32
        self.num_epochs = 25
        self.weight_init = "xavier"
        self.learning_rate = 1e-3
        self.batch_norm = False

        self.loss_type = "l2"

        self.l2_loss = False
        self.alpha = 1e-3

        self.dropout = False
        self.keep_rate = 0.5
<file_sep>/models/cnn_model.py
"""
Model implementation.
"""
from helper import cache_func, INIT_METHODS
import tensorflow as tf


class CNNModel:
    """
    CNN model implementation. Covers the implementations for both the large and the
    compact network.
    """
    def __init__(self, data, target, model_params, data_params):
        self.data = data
        self.target = target
        self.params = model_params
        self.data_params = data_params

        # Weight initialization object arguments.
        if self.params["weight_init"] == "gaussian":
            init_args = {"mean": 0.0,
                         "stddev": 1}
        else:
            init_args = {}

        if model_params["model_type"] == "large":       # LargeNet Implementation.
            self.num_layers = 9                         # Number of conv. layers.
            self.num_deep = 4                           # Number of dense layers.
            self.strides = [1, 1, 1, 1, 1, 1, 1, 1]
            self.weight_dict = {"W_c_1": tf.compat.v1.get_variable(shape=(7, 7, self.data_params["input_dims"], 16),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_1"),
                                "b_c_1": tf.Variable(tf.zeros([16])),
                                "W_c_2": tf.compat.v1.get_variable(shape=(7, 7, 16, 16),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_2"),
                                "b_c_2": tf.Variable(tf.zeros([16])),
                                "W_c_3": tf.compat.v1.get_variable(shape=(5, 5, 16, 32),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_3"),
                                "b_c_3": tf.Variable(tf.zeros([32])),
                                "W_c_4": tf.compat.v1.get_variable(shape=(5, 5, 32, 32),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_4"),
                                "b_c_4": tf.Variable(tf.zeros([32])),
                                "W_c_5": tf.compat.v1.get_variable(shape=(5, 5, 32, 64),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_5"),
                                "b_c_5": tf.Variable(tf.zeros([64])),
                                "W_c_6": tf.compat.v1.get_variable(shape=(3, 3, 64, 128),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_6"),
                                "b_c_6": tf.Variable(tf.zeros([128])),
                                "W_c_7": tf.compat.v1.get_variable(shape=(3, 3, 128, 256),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_7"),
                                "b_c_7": tf.Variable(tf.zeros([256])),
                                "W_c_8": tf.compat.v1.get_variable(shape=(3, 3, 256, 512),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                   name="W_c_8"),
                                "b_c_8": tf.Variable(tf.zeros([512])),
                                "W_1": tf.compat.v1.get_variable(shape=(34 ** 2 * 512, 512),
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_1"),
                                "b_1": tf.Variable(tf.zeros([512])),
                                "W_2": tf.compat.v1.get_variable(shape=[512, 256],
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_2"),
                                "b_2": tf.Variable(tf.zeros([256])),
                                "W_3": tf.compat.v1.get_variable(shape=(256, 1),
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_3"),
                                "b_3": tf.Variable(tf.zeros([1]))}

        else:                           # Implementation of the compact network.
            self.num_layers = 5
            self.num_deep = 4
            self.strides = [3, 1, 1, 1]
            self.weight_dict = {"W_c_1": tf.compat.v1.get_variable(shape=(5, 5, self.data_params["input_dims"], 16),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](
                                                                       **init_args),
                                                                   name="W_c_1"),
                                "b_c_1": tf.Variable(tf.zeros([16])),
                                "W_c_2": tf.compat.v1.get_variable(shape=(5, 5, 16, 32),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](
                                                                       **init_args),
                                                                   name="W_c_2"),
                                "b_c_2": tf.Variable(tf.zeros([32])),
                                "W_c_3": tf.compat.v1.get_variable(shape=(3, 3, 32, 64),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](
                                                                       **init_args),
                                                                   name="W_c_3"),
                                "b_c_3": tf.Variable(tf.zeros([64])),
                                "W_c_4": tf.compat.v1.get_variable(shape=(3, 3, 64, 128),
                                                                   initializer=INIT_METHODS[self.params["weight_init"]](
                                                                       **init_args),
                                                                   name="W_c_4"),
                                "b_c_4": tf.Variable(tf.zeros([128])),
                                "W_1": tf.compat.v1.get_variable(shape=(10 ** 2 * 128, 1024),
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_1"),
                                "b_1": tf.Variable(tf.zeros([1024])),
                                "W_2": tf.compat.v1.get_variable(shape=[1024, 256],
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_2"),
                                "b_2": tf.Variable(tf.zeros([256])),
                                "W_3": tf.compat.v1.get_variable(shape=(256, 1),
                                                                 initializer=INIT_METHODS[self.params["weight_init"]](**init_args),
                                                                 name="W_3"),
                                "b_3": tf.Variable(tf.zeros([1]))}

        for key, val in self.weight_dict.items():   # Rename the layers.
            self.weight_dict[key] = tf.identity(self.weight_dict[key], name=key)

        # Flag indicating whether the current session is training or prediction.
        self.training = tf.compat.v1.placeholder(tf.bool, shape=[])
        self.layer_out = None

        self.convs = {}
        self.biass = {}
        self.activations = {}
        self.pools = {}
        self.weight = {"W_c_1": None}

        self.densed = {}
        self.biased = {}
        self.activated = {}
        self.dense_weights = {}
        self.bias_weights = {}
        self._loss = None

        # Initialize the lazy properties.
        _ = self.optimize
        _ = self.eval_loss
        self.epoch_counter = 0

    @cache_func
    def predict(self):
        """
        Forward function for the models.
        :return: Output of the model.
        """
        for c in range(1, self.num_layers):
            if c == 1:
                input_ = self.data
                self.weight["W_c_1"] = tf.transpose(self.weight_dict["W_c_1"], [3, 0, 1, 2])
            else:
                input_ = pool
            conv = tf.nn.conv2d(input_, self.weight_dict[f"W_c_{c}"], self.strides[c-1], "VALID")
            self.convs[c] = tf.transpose(conv, [3, 0, 1, 2])
            bias = tf.nn.bias_add(conv, self.weight_dict[f"b_c_{c}"])
            self.biass[c] = tf.transpose(bias, [3, 0, 1, 2])
            if self.params["batch_norm"]:
                bias = tf.compat.v1.layers.batch_normalization(bias, axis=-1, training=self.training,
                                                               momentum=0.7)
            activation = tf.nn.relu(bias)
            self.activations[c] = tf.transpose(activation, [3, 0, 1, 2])
            pool = tf.nn.pool(activation, (3, 3),
                              "MAX", padding="VALID")
            self.pools[c] = tf.transpose(pool, [3, 0, 1, 2])

        flatten = tf.compat.v1.layers.flatten(pool, name=None, data_format='channels_last')

        layer_out = flatten
        for d in range(1, self.num_deep):
            densed = tf.matmul(layer_out, self.weight_dict[f"W_{d}"],
                               transpose_a=False, transpose_b=False)
            self.densed[d] = densed
            self.dense_weights[d] = self.weight_dict[f"W_{d}"]
            layer_out = densed + self.weight_dict[f"b_{d}"]
            self.biased[d] = layer_out
            self.bias_weights = self.weight_dict[f"b_{d}"]

            if self.params["batch_norm"]:
                layer_out = tf.compat.v1.layers.batch_normalization(layer_out, axis=-1, training=self.training,
                                                                    momentum=0.7)

            if d != self.num_deep - 1:
                layer_out = tf.nn.relu(layer_out)
                self.activated[d] = layer_out

                if self.params["dropout"]:
                    layer_out = tf.cond(self.training, lambda: tf.nn.dropout(layer_out, self.params["keep_rate"]),
                                        lambda: layer_out)

        return layer_out

    @cache_func
    def optimize(self):
        """
        One step optimization for the specified loss function.
        :return: optimizer.
        """
        loss = self.loss
        optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=self.params["learning_rate"],
                                                     beta1=0.8, beta2=0.8)
        train_op = optimizer.minimize(loss)
        return train_op

    @cache_func
    def loss(self):
        """
        Overall loss function that contains the data loss and the regularization losses.
        :return: Overall loss.
        """
        self._loss = self.data_loss
        if self.params["l2_loss"]:
            self._loss += self.l2_loss
        return self._loss

    @cache_func
    def data_loss(self):
        """
        Data loss from the label predictions.
        :return: data loss.
        """
        if self.params["loss_type"] == "l1":
            loss = tf.reduce_mean(tf.abs(tf.subtract(self.target, self.predict)))
        elif self.params["loss_type"] == "l2":
            loss = tf.reduce_mean(tf.pow(tf.subtract(self.target, self.predict), 2))
        else:
            loss = 0.0
        return loss

    @cache_func
    def eval_loss(self):
        """
        Evaluation loss, L1.
        :return: evaluation loss.
        """
        loss = tf.reduce_mean(tf.abs(tf.subtract(self.target, tf.math.round(self.predict))))
        return loss

    @cache_func
    def l2_loss(self):
        """
        L2 regularization loss.
        :return: Regularization loss.
        """
        l2_loss = 0.0
        for key, val in self.weight_dict.items():
            l2_loss += self.params["alpha"] * tf.nn.l2_loss(val)
        return l2_loss

    def evaluate(self):
        """

        :return:
        """
        return tf.reduce_mean(tf.abs(tf.subtract(self.target, tf.math.round(self.predict))))
<file_sep>/README.md
# Assessing Face Attractivity With Convolutional Networks

## Author
<NAME>

## Folder structure

## images/

This folder contains the input data for all the training, validation and the test splits. Therefore,
this folder has three subfolders:

- **training/**
- **validation/**
- **test/**

In each of those folders, input images are located, which are named with their regression
labels. For example, for the file name "1_AF5.jpg", the regression label is the 1 at the beginning.


### models/

Contains the model implementation script. There is only one model script in this folder. In this scirpt,
forward function for the model and the loss functions are implemented.


## outs/

This is a temporary folder for the checkpoints generated by the model summary functions. These functions
record the model weights and the layer outputs at each iteration. These checkpoints are loaded
by the tensorboard.


### config.py
This script contains the model and data preprocessing parameters. Before running the code, 
parameters will be modified here. Available parameters are as listed below:

- **Model type**: Type of the model to run. In the report, two models are introduced, the
              LargeNet (selected with "large") and the CharmNet (selected with "compact").
              
- **Batch Size**: Size of the batches for the mini-batches.
- **Epoch Numbers**: Number of training epochs.
- **Weight Initialization**: Weight initialization method. Can be either "gaussian" or "xavier".
- **Learning Rate**: Learning rate of the parameters.
- **Batch Normalization**: Flag for toggling the batch normalization layers between the convolutional
                           and the dense layers.
- **Loss Type**: Type of the loss function to be used in the training. Can be "l1" or "l2" for the
                 L1 and L2 losses.
- **L2 Loss**: Flag toggling the L2 weight regularization.
- **alpha**: Loss weight for the regularization loss.
- **Dropout**: Toggles the dropout layers between the dense layers.
- **Keep Rate**: Keep probability for the neurons in the dense layers.                           
         



### data_processor.py

This script contains the implementation of the data processor class. This class reads the images
from the default data folder path which is "./images/". For each of the training, validation or the
test split, all data is read and their labels are stored separately. 

During training, random mini-batches are generated from the whole list of images. These batches are
given to the model for each call. For the validation and the test sets, whole data is returned 
without generating mini-batches.



### helper.py

In this script, we have imported a wrapper function to be used in the model implementation. This
wrapper is the lazy_property written by the

https://danijar.com/structuring-your-tensorflow-models/


### main.py

This is the main script of the project. It does not accept any arguments and takes the run
configurations from the "config.py". Here, the parameter and layer logging functions are initialized.
Moreover, the model and the data are prepared.

Training is done in a main for loop which iterates over the whole data for the specified number of
epochs. 

## How to run?

We are using the Tensorflow 2.2 with the GPU support. In addition, the scikit-image is needed. After
the requirements are installed with the following command,


```bash
pip3 install -r requirements.txt
```

the main script can be run with the following command,

```bash
python3 main.py
```


When the training starts, print statements will inform the user about the training procedure. In addition
to the verbosity provided by the print statements, it is possible to inspect the model weights and
the layer histograms via tensorboard with the following command,

```bash
tensorboard --logdir ./
```
<file_sep>/requirements.txt
absl-py==0.9.0
asn1crypto==0.24.0
astunparse==1.6.3
cachetools==4.0.0
certifi==2019.11.28
chardet==3.0.4
click==7.1.1
cryptography==2.1.4
cycler==0.10.0
decorator==4.4.2
Flask==1.1.1
gast==0.3.3
google-auth==1.12.0
google-auth-oauthlib==0.4.1
google-pasta==0.2.0
grpcio==1.27.2
h5py==2.10.0
idna==2.6
imageio==2.8.0
itsdangerous==1.1.0
Jinja2==2.11.1
Keras-Preprocessing==1.1.0
keyring==10.6.0
keyrings.alt==3.0
kiwisolver==1.1.0
Markdown==3.2.1
MarkupSafe==1.1.1
matplotlib==3.2.1
networkx==2.4
numpy==1.18.2
oauthlib==3.1.0
opt-einsum==3.2.0
Pillow==7.0.0
protobuf==3.11.3
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycrypto==2.6.1
pygobject==3.26.1
pyparsing==2.4.6
python-apt==1.6.5+ubuntu0.2
python-dateutil==2.8.1
PyWavelets==1.1.1
pyxdg==0.25
requests==2.23.0
requests-oauthlib==1.3.0
rsa==4.0
scikit-image==0.16.2
scipy==1.4.1
SecretStorage==2.3.1
six==1.14.0
tensorboard==2.2.0
tensorboard-plugin-wit==1.6.0.post2
tensorflow-estimator==2.2.0rc0
tensorflow-gpu==2.2.0rc2
termcolor==1.1.0
urllib3==1.25.8
Werkzeug==1.0.0
wrapt==1.12.1
<file_sep>/main.py
import os
import numpy as np
import matplotlib.pyplot as plt
import tensorflow.compat.v1 as tf
from data_processor import DataProcessor
from models.cnn_model import CNNModel
import config

tf.disable_v2_behavior()
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

if __name__ == "__main__":
    data_params = config.DataParams().__dict__
    model_params = config.ModelParams().__dict__

    # Data generation for the training, validation and the test.
    data = DataProcessor("train")
    data.init_random_batches(model_params["batch_size"])
    val_data = DataProcessor("val")
    test_data = DataProcessor("test")

    # Placeholders for the model input and output.
    x = tf.placeholder("float", (None, data_params["input_size"],
                                 data_params["input_size"], data_params["input_dims"]))
    y = tf.placeholder("float", (None, 1))

    # Model class and weight initialization. Also, the step initialization for the epochs.
    model = CNNModel(model_params=model_params, data_params=data_params, data=x, target=y)
    global_step = tf.Variable(0, dtype=tf.int32, trainable=False)
    init = tf.global_variables_initializer()

    # Initiation of the main Session.
    with tf.Session() as sess:
        # Summary writer folder.
        file_writer = tf.summary.FileWriter('./outs', sess.graph)
        sess.run(init)
        if not data.has_next():
            print("All data finished")
            data.init_random_batches(model_params["batch_size"])

        # Enable updating through batch normalization and dropout layers.
        extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

        # Namefield descriptions of the parameters that are logged through each iteration.
        with tf.name_scope("Layer-1") as scope:
            tf.summary.image(f"Filters", model.weight[f"W_c_{1}"])  # First conv. layer

        for c in range(1, model.num_layers):
            with tf.name_scope("ConvLayer-" + str(c)) as scope:

                tf.summary.histogram(f"Convolved Image", model.convs[c])        # Image after convolution.
                tf.summary.histogram(f"Filter", model.weight_dict[f"W_c_{1}"])  # Filter histogram.

                tf.summary.histogram(f"Biased Image", model.biass[c])           # Biased image histogram.
                tf.summary.histogram(f"Bias", model.weight_dict[f"b_c_{c}"])    # Bias histogram.

                tf.summary.histogram(f"Activation Image", model.activations[c]) # Image hist. after the activation.
                tf.summary.histogram(f"Pooled Image", model.pools[c])           # Image hist. after the pooling.

        for d in range(1, model.num_deep):
            with tf.name_scope("DenseLayer-" + str(d)) as scope:
                tf.summary.histogram("Dense Output", model.densed[d])           # Output hist. after the weights.
                tf.summary.histogram("Biased Output", model.biased[d])          # Output hist after the bias.
                try:
                    tf.summary.histogram("Activated Output", model.activated[d])    # Output after batch_norm layer.
                except KeyError:
                    pass

        merged_summary_op = tf.summary.merge_all()      # Merges all summaries from different logs.

        for epoch in range(model_params["num_epochs"]):
            global_step += 1
            print("\nEPOCH: " + str(epoch))
            data.init_random_batches(batch_size=model_params["batch_size"])     # Shuffle the training data.
            mean_cost = 0.0
            len_ = 0
            while True:     # For all mini-batches...
                batch_xs, batch_ys = data.next_batches()    # Get the next mini-batch.
                if batch_xs is None:
                    break
                sess.run([model.optimize, extra_update_ops], feed_dict={model.data: batch_xs,
                                                                        model.target: batch_ys,
                                                                        model.training: True})
                # Optimize the model for a single step.

                cost = sess.run(model.eval_loss, feed_dict={model.data: batch_xs,
                                                            model.target: batch_ys,
                                                            model.training: False})
                # Compute the loss function.
                # And print for every batch.
                print(f"\rTraining Loss: {cost}", end="", flush=True)
                mean_cost += cost
                len_ += 1

            # Print the average loss for all batches in the training set.
            print(f"\rTraining Loss: {mean_cost / len_}")
            val_x, val_y = val_data.get_data()
            val_loss, summ = sess.run([model.eval_loss, merged_summary_op], feed_dict={model.data: val_x,
                                                                                       model.target: val_y,
                                                                                       model.training: False})

            file_writer.add_summary(summ, global_step.eval(session=sess))
            file_writer.add_summary(summary=tf.Summary(value=[tf.Summary.Value(tag="Train-Loss",
                                                                               simple_value=mean_cost / len_),
                                                              ]), global_step=global_step.eval(session=sess))
            file_writer.add_summary(summary=tf.Summary(value=[tf.Summary.Value(tag="Reg-Loss",
                                                                               simple_value=model.l2_loss.eval()),
                                                              ]), global_step=global_step.eval(session=sess))
            file_writer.add_summary(summary=tf.Summary(value=[tf.Summary.Value(tag="Val-Loss", simple_value=val_loss),
                                                       ]), global_step=global_step.eval(session=sess))

            print(f"Validation Loss: {val_loss}")
            file_writer.flush()

        test_x, test_y = test_data.get_data()
        test_loss = sess.run([model.eval_loss], feed_dict={model.data: test_x,
                                                           model.target: test_y,
                                                           model.training: False})

        print("Test Loss: " + str(test_loss))

        random_indices = np.random.choice(list(range(test_x.shape[0])), 9)
        random_images = [test_x[idx][None, :] for idx in random_indices]
        random_preds = [sess.run(model.predict, feed_dict={model.data: image, model.training: False})
                        for image in random_images]
        random_labels = [test_y[idx] for idx in random_indices]

        for i, image, prediction, label in zip(range(9), random_images, random_preds, random_labels):
            plt.subplot(3, 3, i+1)
            plt.tight_layout()
            plt.imshow(np.mean(((image[0] - image.min()) / (image.max() - image.min()) * 255), axis=-1).astype(int))
            plt.title("True Score: {}".format(label))
            plt.xlabel("Model prediction: \n{}".format(prediction[0]))

        plt.savefig("./samples.png", figsize=(70, 70))
<file_sep>/data_processor.py
"""
Data processing class. Reads the raw data from the folder and
generates mini-batches and labels.
"""
from skimage import transform, io
import numpy as np
import pathlib
import random
import time


class DataProcessor:
    """
    Data processing class implementation.
    """
    def __init__(self, data_type):
        """
        Initialization.
        :param data_type: Type of the split of the data. Choose train, validation or test.
        """
        if data_type == "train":
            self.data_list = list(pathlib.Path("images/training/").glob('*'))
        elif data_type == "validation":
            self.data_list = list(pathlib.Path("images/validation/").glob('*'))
        else:
            self.data_list = list(pathlib.Path("images/test/").glob('*'))

        self.data = []
        self.label = []

        self.__read_data_to_tensor()
        self.data_copy = []
        self.label_copy = []

        self.middle_data = None
        self.middle_label = None

        self.cursor = 0

    def get_data(self):
        """
        Stacks the list of data and labels read from the files.
        :return: Tensor of images.
        """
        return np.stack(self.data, axis=0), np.stack(self.label, axis=0)[:, None]

    def init_random_batches(self, batch_size):
        """
        Creates random mini-batches from the read and transformed data.
        :param batch_size: Size of the random mini-batches.
        :return: None.
        """
        self.__reset_data()
        self.cursor = 0
        index = self.__get_index(self.data)
        random.shuffle(index)
        self.middle_data = list()
        self.middle_label = list()
        for item in range(len(index)):
            self.middle_data.append(self.data[index[item]])
            self.middle_label.append(self.label[index[item]])

        self.data_copy = self.__divide_batches(self.middle_data, batch_size)
        self.label_copy = self.__divide_batches(self.middle_label, batch_size)

    def next_batches(self):
        """
        Returns the next mini-batch. If there are no mini-batches left, returns None.
        :return: (batch_x, batch_y)
        """
        self.cursor += 1
        if self.has_next():
            return self.data_copy[self.cursor - 1], np.array(self.label_copy[self.cursor - 1])[:, None]
        else:
            return None, None

    def has_next(self):
        """
        Flag to check whether there are any more batches to return.
        :return: Flag.
        """
        return self.cursor != len(self.data_copy)

    def __reset_data(self):
        """
        Stores the current data to an additional array.
        :return: None.
        """
        self.data_copy = self.data.copy()
        self.label_copy = self.label.copy()

    def __read_data_to_tensor(self):
        """
        Reads the raw data from files to a single list of tensors.
        :return: None.
        """
        start_time = time.time()
        for image_path in self.data_list:
            self.data.append(self.__decode_img(io.imread(str(image_path))))
            self.label.append(self.__get_label(image_path.stem))

        whole_data = np.stack(self.data, axis=0)
        data_mean = whole_data.mean(axis=0)
        data_std = whole_data.std(axis=0)
        norm_data = (whole_data - data_mean) / data_std
        self.data = [data for data in norm_data]
        print("Reading images takes %s\n" % (time.time() - start_time))

    @staticmethod
    def __get_index(data):
        """
        Returns the indices of the current data list.
        :param data: list of image tensors.
        :return: List of indices.
        """
        return list(range(len(data)))

    @staticmethod
    def __divide_batches(data, batch_size):
        """
        Divides and returns the batches for the specified batch size.
        :param data: List of image tensors.
        :param batch_size: Batch size for mini-batches.
        :return: Mini-batch.
        """
        data = np.stack(data, axis=0)
        return [data[i:i + batch_size] for i in range(0, int(data.shape[0]), batch_size - 1)]

    @staticmethod
    def __decode_img(img):
        """
        Decodes the image arrays to float type with fixed size of 80x80.
        :param img: Image array.
        :return: Decoded image array.
        """
        img = img.astype(float)
        return transform.resize(img, [80, 80])

    @staticmethod
    def __get_label(file_path):
        """
        Reads the label from the corresponding file path of the image.
        :param file_path: File path of the image.
        :return: label.
        """
        parts = file_path.split("_")[0]
        return parts
