Upload 10 files

build_vocab.py

@@ -0,0 +1,80 @@
+import pickle
+from collections import Counter
+import json
+
+
+class JsonReader(object):
+    def __init__(self, json_file):
+        self.data = self.__read_json(json_file)
+        self.keys = list(self.data.keys())
+
+    def __read_json(self, filename):
+        with open(filename, 'r') as f:
+            data = json.load(f)
+        return data
+
+    def __getitem__(self, item):
+        return self.data[item]
+        # return self.data[self.keys[item]]
+
+    def __len__(self):
+        return len(self.data)
+
+
+class Vocabulary(object):
+    def __init__(self):
+        self.word2idx = {}
+        self.id2word = {}
+        self.idx = 0
+        self.add_word('<pad>')
+        self.add_word('<end>')
+        self.add_word('<start>')
+        self.add_word('<unk>')
+
+    def add_word(self, word):
+        if word not in self.word2idx:
+            self.word2idx[word] = self.idx
+            self.id2word[self.idx] = word
+            self.idx += 1
+
+    def get_word_by_id(self, id):
+        return self.id2word[id]
+
+    def __call__(self, word):
+        if word not in self.word2idx:
+            return self.word2idx['<unk>']
+        return self.word2idx[word]
+
+    def __len__(self):
+        return len(self.word2idx)
+
+
+def build_vocab(json_file, threshold):
+    caption_reader = JsonReader(json_file)
+    counter = Counter()
+
+    for items in caption_reader:
+        text = items.replace('.', '').replace(',', '')
+        counter.update(text.lower().split(' '))
+    words = [word for word, cnt in counter.items() if cnt > threshold and word != '']
+    vocab = Vocabulary()
+
+    for word in words:
+        print(word)
+        vocab.add_word(word)
+    return vocab
+
+
+def main(json_file, threshold, vocab_path):
+    vocab = build_vocab(json_file=json_file,
+                        threshold=threshold)
+    with open(vocab_path, 'wb') as f:
+        pickle.dump(vocab, f)
+    print("Total vocabulary size:{}".format(len(vocab)))
+    print("Saved path in {}".format(vocab_path))
+
+
+if __name__ == '__main__':
+    main(json_file='../data/new_data/debugging_captions.json',
+         threshold=0,
+         vocab_path='../data/new_data/debug_vocab.pkl')

callbacks.py

@@ -0,0 +1,1066 @@
+"""Callbacks: utilities called at certain points during model training.
+
+# Adapted from
+
+ - https://github.com/keras-team/keras
+ - https://github.com/bstriner/keras-tqdm/blob/master/keras_tqdm/tqdm_callback.py
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import csv
+import six
+
+import numpy as np
+import time
+import json
+import warnings
+from tqdm import tqdm
+
+from collections import deque
+from collections import OrderedDict
+from collections import Iterable
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+
+class CallbackList(object):
+    """Container abstracting a list of callbacks.
+
+    # Arguments
+        callbacks: List of `Callback` instances.
+        queue_length: Queue length for keeping
+            running statistics over callback execution time.
+    """
+
+    def __init__(self, callbacks=None, queue_length=10):
+        callbacks = callbacks or []
+        self.callbacks = [c for c in callbacks]
+        self.queue_length = queue_length
+
+    def append(self, callback):
+        self.callbacks.append(callback)
+
+    def set_params(self, params):
+        for callback in self.callbacks:
+            callback.set_params(params)
+
+    def set_model(self, model):
+        for callback in self.callbacks:
+            callback.set_model(model)
+
+    def on_epoch_begin(self, epoch, logs=None):
+        """Called at the start of an epoch.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        for callback in self.callbacks:
+            callback.on_epoch_begin(epoch, logs)
+        self._delta_t_batch = 0.
+        self._delta_ts_batch_begin = deque([], maxlen=self.queue_length)
+        self._delta_ts_batch_end = deque([], maxlen=self.queue_length)
+
+    def on_epoch_end(self, epoch, logs=None):
+        """Called at the end of an epoch.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        for callback in self.callbacks:
+            callback.on_epoch_end(epoch, logs)
+
+    def on_batch_begin(self, batch, logs=None):
+        """Called right before processing a batch.
+
+        # Arguments
+            batch: integer, index of batch within the current epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        t_before_callbacks = time.time()
+        for callback in self.callbacks:
+            callback.on_batch_begin(batch, logs)
+        self._delta_ts_batch_begin.append(time.time() - t_before_callbacks)
+        delta_t_median = np.median(self._delta_ts_batch_begin)
+        if (self._delta_t_batch > 0. and
+           delta_t_median > 0.95 * self._delta_t_batch and
+           delta_t_median > 0.1):
+            warnings.warn('Method on_batch_begin() is slow compared '
+                          'to the batch update (%f). Check your callbacks.'
+                          % delta_t_median)
+        self._t_enter_batch = time.time()
+
+    def on_batch_end(self, batch, logs=None):
+        """Called at the end of a batch.
+
+        # Arguments
+            batch: integer, index of batch within the current epoch.
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        if not hasattr(self, '_t_enter_batch'):
+            self._t_enter_batch = time.time()
+        self._delta_t_batch = time.time() - self._t_enter_batch
+        t_before_callbacks = time.time()
+        for callback in self.callbacks:
+            callback.on_batch_end(batch, logs)
+        self._delta_ts_batch_end.append(time.time() - t_before_callbacks)
+        delta_t_median = np.median(self._delta_ts_batch_end)
+        if (self._delta_t_batch > 0. and
+           (delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1)):
+            warnings.warn('Method on_batch_end() is slow compared '
+                          'to the batch update (%f). Check your callbacks.'
+                          % delta_t_median)
+
+    def on_train_begin(self, logs=None):
+        """Called at the beginning of training.
+
+        # Arguments
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        for callback in self.callbacks:
+            callback.on_train_begin(logs)
+
+    def on_train_end(self, logs=None):
+        """Called at the end of training.
+
+        # Arguments
+            logs: dictionary of logs.
+        """
+        logs = logs or {}
+        for callback in self.callbacks:
+            callback.on_train_end(logs)
+
+    def __iter__(self):
+        return iter(self.callbacks)
+
+
+class Callback(object):
+    """Abstract base class used to build new callbacks.
+
+    # Properties
+        params: dict. Training parameters
+            (eg. verbosity, batch size, number of epochs...).
+        model: instance of `keras.models.Model`.
+            Reference of the model being trained.
+
+    The `logs` dictionary that callback methods
+    take as argument will contain keys for quantities relevant to
+    the current batch or epoch.
+
+    Currently, the `.fit()` method of the `Sequential` model class
+    will include the following quantities in the `logs` that
+    it passes to its callbacks:
+
+        on_epoch_end: logs include `acc` and `loss`, and
+            optionally include `val_loss`
+            (if validation is enabled in `fit`), and `val_acc`
+            (if validation and accuracy monitoring are enabled).
+        on_batch_begin: logs include `size`,
+            the number of samples in the current batch.
+        on_batch_end: logs include `loss`, and optionally `acc`
+            (if accuracy monitoring is enabled).
+    """
+
+    def __init__(self):
+        self.validation_data = None
+        self.model = None
+
+    def set_params(self, params):
+        self.params = params
+
+    def set_model(self, model):
+        self.model = model
+
+    def on_epoch_begin(self, epoch, logs=None):
+        pass
+
+    def on_epoch_end(self, epoch, logs=None):
+        pass
+
+    def on_batch_begin(self, batch, logs=None):
+        pass
+
+    def on_batch_end(self, batch, logs=None):
+        pass
+
+    def on_train_begin(self, logs=None):
+        pass
+
+    def on_train_end(self, logs=None):
+        pass
+
+
+class BaseLogger(Callback):
+    """Callback that accumulates epoch averages of metrics.
+
+    This callback is automatically applied to every Keras model.
+    """
+
+    def on_epoch_begin(self, epoch, logs=None):
+        self.seen = 0
+        self.totals = {}
+
+    def on_batch_end(self, batch, logs=None):
+        logs = logs or {}
+        batch_size = logs.get('size', 0)
+        self.seen += batch_size
+
+        for k, v in logs.items():
+            if k in self.totals:
+                self.totals[k] += v * batch_size
+            else:
+                self.totals[k] = v * batch_size
+
+    def on_epoch_end(self, epoch, logs=None):
+        if logs is not None:
+            for k in self.params['metrics']:
+                if k in self.totals:
+                    # Make value available to next callbacks.
+                    logs[k] = self.totals[k] / self.seen
+
+
+class TerminateOnNaN(Callback):
+    """Callback that terminates training when a NaN loss is encountered.
+    """
+
+    def __init__(self):
+        super(TerminateOnNaN, self).__init__()
+
+    def on_batch_end(self, batch, logs=None):
+        logs = logs or {}
+        loss = logs.get('loss')
+        if loss is not None:
+            if np.isnan(loss) or np.isinf(loss):
+                print('Batch %d: Invalid loss, terminating training' % (batch))
+                self.model.stop_training = True
+
+
+class History(Callback):
+    """Callback that records events into a `History` object.
+
+    This callback is automatically applied to
+    every Keras model. The `History` object
+    gets returned by the `fit` method of models.
+    """
+
+    def on_train_begin(self, logs=None):
+        self.epoch = []
+        self.history = {}
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+        self.epoch.append(epoch)
+        for k, v in logs.items():
+            self.history.setdefault(k, []).append(v)
+
+
+class ModelCheckpoint(Callback):
+    """Save the model after every epoch.
+
+    `filepath` can contain named formatting options,
+    which will be filled the value of `epoch` and
+    keys in `logs` (passed in `on_epoch_end`).
+
+    For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
+    then the model checkpoints will be saved with the epoch number and
+    the validation loss in the filename.
+
+    # Arguments
+        filepath: string, path to save the model file.
+        monitor: quantity to monitor.
+        verbose: verbosity mode, 0 or 1.
+        save_best_only: if `save_best_only=True`,
+            the latest best model according to
+            the quantity monitored will not be overwritten.
+        mode: one of {auto, min, max}.
+            If `save_best_only=True`, the decision
+            to overwrite the current save file is made
+            based on either the maximization or the
+            minimization of the monitored quantity. For `val_acc`,
+            this should be `max`, for `val_loss` this should
+            be `min`, etc. In `auto` mode, the direction is
+            automatically inferred from the name of the monitored quantity.
+        save_weights_only: if True, then only the model's weights will be
+            saved (`torch.save(self.model.state_dict(), filepath)`), else the full model
+            is saved (`torch.save(self.model.state_dict(), filepath)`).
+        period: Interval (number of epochs) between checkpoints.
+    """
+
+    def __init__(self, filepath, monitor='val_loss', verbose=0,
+                 save_best_only=False, save_weights_only=False,
+                 mode='auto', period=1):
+        super(ModelCheckpoint, self).__init__()
+        self.monitor = monitor
+        self.verbose = verbose
+        self.filepath = filepath
+        self.save_best_only = save_best_only
+        self.save_weights_only = save_weights_only
+        self.period = period
+        self.epochs_since_last_save = 0
+
+        if mode not in ['auto', 'min', 'max']:
+            warnings.warn('ModelCheckpoint mode %s is unknown, '
+                          'fallback to auto mode.' % (mode),
+                          RuntimeWarning)
+            mode = 'auto'
+
+        if mode == 'min':
+            self.monitor_op = np.less
+            self.best = np.Inf
+        elif mode == 'max':
+            self.monitor_op = np.greater
+            self.best = -np.Inf
+        else:
+            if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):
+                self.monitor_op = np.greater
+                self.best = -np.Inf
+            else:
+                self.monitor_op = np.less
+                self.best = np.Inf
+
+    def on_epoch_end(self, epoch, logs=None):
+        import torch
+        logs = logs or {}
+        self.epochs_since_last_save += 1
+        if self.epochs_since_last_save >= self.period:
+            self.epochs_since_last_save = 0
+            filepath = self.filepath.format(epoch=epoch + 1, **logs)
+            if self.save_best_only:
+                current = logs.get(self.monitor)
+                if current is None:
+                    warnings.warn('Can save best model only with %s available, '
+                                  'skipping.' % (self.monitor), RuntimeWarning)
+                else:
+                    if self.monitor_op(current, self.best):
+                        if self.verbose > 0:
+                            print('\nEpoch %05d: %s improved from %0.5f to %0.5f,'
+                                  ' saving model to %s'
+                                  % (epoch + 1, self.monitor, self.best,
+                                     current, filepath))
+                        self.best = current
+                        if self.save_weights_only:
+                            torch.save(self.model.state_dict(), filepath)
+                        else:
+                            torch.save(self.model.state_dict(), filepath)
+                    else:
+                        if self.verbose > 0:
+                            print('\nEpoch %05d: %s did not improve' %
+                                  (epoch + 1, self.monitor))
+            else:
+                if self.verbose > 0:
+                    print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))
+                if self.save_weights_only:
+                    torch.save(self.model.state_dict(), filepath)
+                else:
+                    torch.save(self.model.state_dict(), filepath)
+
+
+class EarlyStopping(Callback):
+    """Stop training when a monitored quantity has stopped improving.
+
+    # Arguments
+        monitor: quantity to be monitored.
+        min_delta: minimum change in the monitored quantity
+            to qualify as an improvement, i.e. an absolute
+            change of less than min_delta, will count as no
+            improvement.
+        patience: number of epochs with no improvement
+            after which training will be stopped.
+        verbose: verbosity mode.
+        mode: one of {auto, min, max}. In `min` mode,
+            training will stop when the quantity
+            monitored has stopped decreasing; in `max`
+            mode it will stop when the quantity
+            monitored has stopped increasing; in `auto`
+            mode, the direction is automatically inferred
+            from the name of the monitored quantity.
+    """
+
+    def __init__(self, monitor='val_loss',
+                 min_delta=0, patience=0, verbose=0, mode='auto'):
+        super(EarlyStopping, self).__init__()
+
+        self.monitor = monitor
+        self.patience = patience
+        self.verbose = verbose
+        self.min_delta = min_delta
+        self.wait = 0
+        self.stopped_epoch = 0
+
+        if mode not in ['auto', 'min', 'max']:
+            warnings.warn('EarlyStopping mode %s is unknown, '
+                          'fallback to auto mode.' % mode,
+                          RuntimeWarning)
+            mode = 'auto'
+
+        if mode == 'min':
+            self.monitor_op = np.less
+        elif mode == 'max':
+            self.monitor_op = np.greater
+        else:
+            if 'acc' in self.monitor:
+                self.monitor_op = np.greater
+            else:
+                self.monitor_op = np.less
+
+        if self.monitor_op == np.greater:
+            self.min_delta *= 1
+        else:
+            self.min_delta *= -1
+
+    def on_train_begin(self, logs=None):
+        # Allow instances to be re-used
+        self.wait = 0
+        self.stopped_epoch = 0
+        self.best = np.Inf if self.monitor_op == np.less else -np.Inf
+
+    def on_epoch_end(self, epoch, logs=None):
+        current = logs.get(self.monitor)
+        if current is None:
+            warnings.warn(
+                'Early stopping conditioned on metric `%s` '
+                'which is not available. Available metrics are: %s' %
+                (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning
+            )
+            return
+        if self.monitor_op(current - self.min_delta, self.best):
+            self.best = current
+            self.wait = 0
+        else:
+            self.wait += 1
+            if self.wait >= self.patience:
+                self.stopped_epoch = epoch
+                self.model.stop_training = True
+
+    def on_train_end(self, logs=None):
+        if self.stopped_epoch > 0 and self.verbose > 0:
+            print('Epoch %05d: early stopping' % (self.stopped_epoch + 1))
+
+
+class RemoteMonitor(Callback):
+    """Callback used to stream events to a server.
+
+    Requires the `requests` library.
+    Events are sent to `root + '/publish/epoch/end/'` by default. Calls are
+    HTTP POST, with a `images` argument which is a
+    JSON-encoded dictionary of event images.
+
+    # Arguments
+        root: String; root url of the target server.
+        path: String; path relative to `root` to which the events will be sent.
+        field: String; JSON field under which the images will be stored.
+        headers: Dictionary; optional custom HTTP headers.
+    """
+
+    def __init__(self,
+                 root='http://localhost:9000',
+                 path='/publish/epoch/end/',
+                 field='images',
+                 headers=None):
+        super(RemoteMonitor, self).__init__()
+
+        self.root = root
+        self.path = path
+        self.field = field
+        self.headers = headers
+
+    def on_epoch_end(self, epoch, logs=None):
+        if requests is None:
+            raise ImportError('RemoteMonitor requires '
+                              'the `requests` library.')
+        logs = logs or {}
+        send = {}
+        send['epoch'] = epoch
+        for k, v in logs.items():
+            if isinstance(v, (np.ndarray, np.generic)):
+                send[k] = v.item()
+            else:
+                send[k] = v
+        try:
+            requests.post(self.root + self.path,
+                          {self.field: json.dumps(send)},
+                          headers=self.headers)
+        except requests.exceptions.RequestException:
+            warnings.warn('Warning: could not reach RemoteMonitor '
+                          'root server at ' + str(self.root))
+
+
+class TensorBoard(Callback):
+    """TensorBoard basic visualizations.
+
+    [TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard)
+    is a visualization tool provided with TensorFlow.
+
+    This callback writes a log for TensorBoard, which allows
+    you to visualize dynamic graphs of your training and test
+    metrics, as well as activation histograms for the different
+    layers in your model.
+
+    If you have installed TensorFlow with pip, you should be able
+    to launch TensorBoard from the command line:
+    ```sh
+    tensorboard --logdir=/full_path_to_your_logs
+    ```
+
+    When using a backend other than TensorFlow, TensorBoard will still work
+    (if you have TensorFlow installed), but the only feature available will
+    be the display of the losses and metrics plots.
+
+    # Arguments
+        log_dir: the path of the directory where to save the log
+            files to be parsed by TensorBoard.
+        histogram_freq: frequency (in epochs) at which to compute activation
+            and weight histograms for the layers of the model. If set to 0,
+            histograms won't be computed. Validation images (or split) must be
+            specified for histogram visualizations.
+        write_graph: whether to visualize the graph in TensorBoard.
+            The log file can become quite large when
+            write_graph is set to True.
+        write_grads: whether to visualize gradient histograms in TensorBoard.
+            `histogram_freq` must be greater than 0.
+        batch_size: size of batch of inputs to feed to the network
+            for histograms computation.
+        write_images: whether to write model weights to visualize as
+            image in TensorBoard.
+        embeddings_freq: frequency (in epochs) at which selected embedding
+            layers will be saved.
+        embeddings_layer_names: a list of names of layers to keep eye on. If
+            None or empty list all the embedding layer will be watched.
+        embeddings_metadata: a dictionary which maps layer name to a file name
+            in which metadata for this embedding layer is saved. See the
+            [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)
+            about metadata files format. In case if the same metadata file is
+            used for all embedding layers, string can be passed.
+    """
+
+    def __init__(self, log_dir='./logs',
+                 histogram_freq=0,
+                 batch_size=32,
+                 write_graph=True,
+                 write_grads=False,
+                 write_images=False,
+                 embeddings_freq=0,
+                 embeddings_layer_names=None,
+                 embeddings_metadata=None):
+        super(TensorBoard, self).__init__()
+        global tf, projector
+        try:
+            import tensorflow as tf
+            from tensorflow.contrib.tensorboard.plugins import projector
+        except ImportError:
+            raise ImportError('You need the TensorFlow module installed to use TensorBoard.')
+
+        if K.backend() != 'tensorflow':
+            if histogram_freq != 0:
+                warnings.warn('You are not using the TensorFlow backend. '
+                              'histogram_freq was set to 0')
+                histogram_freq = 0
+            if write_graph:
+                warnings.warn('You are not using the TensorFlow backend. '
+                              'write_graph was set to False')
+                write_graph = False
+            if write_images:
+                warnings.warn('You are not using the TensorFlow backend. '
+                              'write_images was set to False')
+                write_images = False
+            if embeddings_freq != 0:
+                warnings.warn('You are not using the TensorFlow backend. '
+                              'embeddings_freq was set to 0')
+                embeddings_freq = 0
+
+        self.log_dir = log_dir
+        self.histogram_freq = histogram_freq
+        self.merged = None
+        self.write_graph = write_graph
+        self.write_grads = write_grads
+        self.write_images = write_images
+        self.embeddings_freq = embeddings_freq
+        self.embeddings_layer_names = embeddings_layer_names
+        self.embeddings_metadata = embeddings_metadata or {}
+        self.batch_size = batch_size
+
+    def set_model(self, model):
+        self.model = model
+        if K.backend() == 'tensorflow':
+            self.sess = K.get_session()
+        if self.histogram_freq and self.merged is None:
+            for layer in self.model.layers:
+
+                for weight in layer.weights:
+                    mapped_weight_name = weight.name.replace(':', '_')
+                    tf.summary.histogram(mapped_weight_name, weight)
+                    if self.write_grads:
+                        grads = model.optimizer.get_gradients(model.total_loss,
+                                                              weight)
+
+                        def is_indexed_slices(grad):
+                            return type(grad).__name__ == 'IndexedSlices'
+                        grads = [
+                            grad.values if is_indexed_slices(grad) else grad
+                            for grad in grads]
+                        tf.summary.histogram('{}_grad'.format(mapped_weight_name), grads)
+                    if self.write_images:
+                        w_img = tf.squeeze(weight)
+                        shape = K.int_shape(w_img)
+                        if len(shape) == 2:  # dense layer kernel case
+                            if shape[0] > shape[1]:
+                                w_img = tf.transpose(w_img)
+                                shape = K.int_shape(w_img)
+                            w_img = tf.reshape(w_img, [1,
+                                                       shape[0],
+                                                       shape[1],
+                                                       1])
+                        elif len(shape) == 3:  # convnet case
+                            if K.image_data_format() == 'channels_last':
+                                # switch to channels_first to display
+                                # every kernel as a separate image
+                                w_img = tf.transpose(w_img, perm=[2, 0, 1])
+                                shape = K.int_shape(w_img)
+                            w_img = tf.reshape(w_img, [shape[0],
+                                                       shape[1],
+                                                       shape[2],
+                                                       1])
+                        elif len(shape) == 1:  # bias case
+                            w_img = tf.reshape(w_img, [1,
+                                                       shape[0],
+                                                       1,
+                                                       1])
+                        else:
+                            # not possible to handle 3D convnets etc.
+                            continue
+
+                        shape = K.int_shape(w_img)
+                        assert len(shape) == 4 and shape[-1] in [1, 3, 4]
+                        tf.summary.image(mapped_weight_name, w_img)
+
+                if hasattr(layer, 'output'):
+                    tf.summary.histogram('{}_out'.format(layer.name),
+                                         layer.output)
+        self.merged = tf.summary.merge_all()
+
+        if self.write_graph:
+            self.writer = tf.summary.FileWriter(self.log_dir,
+                                                self.sess.graph)
+        else:
+            self.writer = tf.summary.FileWriter(self.log_dir)
+
+        if self.embeddings_freq:
+            embeddings_layer_names = self.embeddings_layer_names
+
+            if not embeddings_layer_names:
+                embeddings_layer_names = [layer.name for layer in self.model.layers
+                                          if type(layer).__name__ == 'Embedding']
+
+            embeddings = {layer.name: layer.weights[0]
+                          for layer in self.model.layers
+                          if layer.name in embeddings_layer_names}
+
+            self.saver = tf.train.Saver(list(embeddings.values()))
+
+            embeddings_metadata = {}
+
+            if not isinstance(self.embeddings_metadata, str):
+                embeddings_metadata = self.embeddings_metadata
+            else:
+                embeddings_metadata = {layer_name: self.embeddings_metadata
+                                       for layer_name in embeddings.keys()}
+
+            config = projector.ProjectorConfig()
+            self.embeddings_ckpt_path = os.path.join(self.log_dir,
+                                                     'keras_embedding.ckpt')
+
+            for layer_name, tensor in embeddings.items():
+                embedding = config.embeddings.add()
+                embedding.tensor_name = tensor.name
+
+                if layer_name in embeddings_metadata:
+                    embedding.metadata_path = embeddings_metadata[layer_name]
+
+            projector.visualize_embeddings(self.writer, config)
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+
+        if not self.validation_data and self.histogram_freq:
+            raise ValueError('If printing histograms, validation_data must be '
+                             'provided, and cannot be a generator.')
+        if self.validation_data and self.histogram_freq:
+            if epoch % self.histogram_freq == 0:
+
+                val_data = self.validation_data
+                tensors = (self.model.inputs +
+                           self.model.targets +
+                           self.model.sample_weights)
+
+                if self.model.uses_learning_phase:
+                    tensors += [K.learning_phase()]
+
+                assert len(val_data) == len(tensors)
+                val_size = val_data[0].shape[0]
+                i = 0
+                while i < val_size:
+                    step = min(self.batch_size, val_size - i)
+                    if self.model.uses_learning_phase:
+                        # do not slice the learning phase
+                        batch_val = [x[i:i + step] for x in val_data[:-1]]
+                        batch_val.append(val_data[-1])
+                    else:
+                        batch_val = [x[i:i + step] for x in val_data]
+                    assert len(batch_val) == len(tensors)
+                    feed_dict = dict(zip(tensors, batch_val))
+                    result = self.sess.run([self.merged], feed_dict=feed_dict)
+                    summary_str = result[0]
+                    self.writer.add_summary(summary_str, epoch)
+                    i += self.batch_size
+
+        if self.embeddings_freq and self.embeddings_ckpt_path:
+            if epoch % self.embeddings_freq == 0:
+                self.saver.save(self.sess,
+                                self.embeddings_ckpt_path,
+                                epoch)
+
+        for name, value in logs.items():
+            if name in ['batch', 'size']:
+                continue
+            summary = tf.Summary()
+            summary_value = summary.value.add()
+            summary_value.simple_value = value.item()
+            summary_value.tag = name
+            self.writer.add_summary(summary, epoch)
+        self.writer.flush()
+
+    def on_train_end(self, _):
+        self.writer.close()
+
+
+class CSVLogger(Callback):
+    """Callback that streams epoch results to a csv file.
+
+    Supports all values that can be represented as a string,
+    including 1D iterables such as np.ndarray.
+
+    # Example
+
+    ```python
+    csv_logger = CSVLogger('training.log')
+    model.fit(X_train, Y_train, callbacks=[csv_logger])
+    ```
+
+    # Arguments
+        filename: filename of the csv file, e.g. 'run/log.csv'.
+        separator: string used to separate elements in the csv file.
+        append: True: append if file exists (useful for continuing
+            training). False: overwrite existing file,
+        output_on_train_end: An additional output file to write to
+            write to when training ends. An example is
+            CSVLogger(filename='./mylog.csv', output_on_train_end=os.sys.stdout)
+    """
+
+    def __init__(self, filename, separator=',', append=False, output_on_train_end=None):
+        self.sep = separator
+        self.filename = filename
+        self.append = append
+        self.writer = None
+        self.keys = None
+        self.append_header = True
+        self.file_flags = 'b' if six.PY2 and os.name == 'nt' else ''
+        self.output_on_train_end = output_on_train_end
+        super(CSVLogger, self).__init__()
+
+    def on_train_begin(self, logs=None):
+        if self.append:
+            if os.path.exists(self.filename):
+                with open(self.filename, 'r' + self.file_flags) as f:
+                    self.append_header = not bool(len(f.readline()))
+            self.csv_file = open(self.filename, 'a' + self.file_flags)
+        else:
+            self.csv_file = open(self.filename, 'w' + self.file_flags)
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+
+        def handle_value(k):
+            is_zero_dim_ndarray = isinstance(k, np.ndarray) and k.ndim == 0
+            if isinstance(k, six.string_types):
+                return k
+            elif isinstance(k, Iterable) and not is_zero_dim_ndarray:
+                return '"[%s]"' % (', '.join(map(str, k)))
+            else:
+                return k
+
+        if self.keys is None:
+            self.keys = sorted(logs.keys())
+
+        if self.model is not None and getattr(self.model, 'stop_training', False):
+            # We set NA so that csv parsers do not fail for this last epoch.
+            logs = dict([(k, logs[k]) if k in logs else (k, 'NA') for k in self.keys])
+
+        if not self.writer:
+            class CustomDialect(csv.excel):
+                delimiter = self.sep
+
+            self.writer = csv.DictWriter(self.csv_file,
+                                         fieldnames=['epoch'] + self.keys, dialect=CustomDialect)
+            if self.append_header:
+                self.writer.writeheader()
+
+        row_dict = OrderedDict({'epoch': epoch})
+        row_dict.update((key, handle_value(logs[key])) for key in self.keys)
+        self.writer.writerow(row_dict)
+        self.csv_file.flush()
+
+    def on_train_end(self, logs=None):
+        self.csv_file.close()
+        if os.path.exists(self.filename):
+            with open(self.filename, 'r' + self.file_flags) as f:
+                print(f.read(), file=self.output_on_train_end)
+        self.writer = None
+
+
+class LambdaCallback(Callback):
+    r"""Callback for creating simple, custom callbacks on-the-fly.
+
+    This callback is constructed with anonymous functions that will be called
+    at the appropriate time. Note that the callbacks expects positional
+    arguments, as:
+
+     - `on_epoch_begin` and `on_epoch_end` expect two positional arguments:
+        `epoch`, `logs`
+     - `on_batch_begin` and `on_batch_end` expect two positional arguments:
+        `batch`, `logs`
+     - `on_train_begin` and `on_train_end` expect one positional argument:
+        `logs`
+
+    # Arguments
+        on_epoch_begin: called at the beginning of every epoch.
+        on_epoch_end: called at the end of every epoch.
+        on_batch_begin: called at the beginning of every batch.
+        on_batch_end: called at the end of every batch.
+        on_train_begin: called at the beginning of model training.
+        on_train_end: called at the end of model training.
+
+    # Example
+
+    ```python
+    # Print the batch number at the beginning of every batch.
+    batch_print_callback = LambdaCallback(
+        on_batch_begin=lambda batch,logs: print(batch))
+
+    # Stream the epoch loss to a file in JSON format. The file content
+    # is not well-formed JSON but rather has a JSON object per line.
+    import json
+    json_log = open('loss_log.json', mode='wt', buffering=1)
+    json_logging_callback = LambdaCallback(
+        on_epoch_end=lambda epoch, logs: json_log.write(
+            json.dumps({'epoch': epoch, 'loss': logs['loss']}) + '\n'),
+        on_train_end=lambda logs: json_log.close()
+    )
+
+    # Terminate some processes after having finished model training.
+    processes = ...
+    cleanup_callback = LambdaCallback(
+        on_train_end=lambda logs: [
+            p.terminate() for p in processes if p.is_alive()])
+
+    model.fit(...,
+              callbacks=[batch_print_callback,
+                         json_logging_callback,
+                         cleanup_callback])
+    ```
+    """
+
+    def __init__(self,
+                 on_epoch_begin=None,
+                 on_epoch_end=None,
+                 on_batch_begin=None,
+                 on_batch_end=None,
+                 on_train_begin=None,
+                 on_train_end=None,
+                 **kwargs):
+        super(LambdaCallback, self).__init__()
+        self.__dict__.update(kwargs)
+        if on_epoch_begin is not None:
+            self.on_epoch_begin = on_epoch_begin
+        else:
+            self.on_epoch_begin = lambda epoch, logs: None
+        if on_epoch_end is not None:
+            self.on_epoch_end = on_epoch_end
+        else:
+            self.on_epoch_end = lambda epoch, logs: None
+        if on_batch_begin is not None:
+            self.on_batch_begin = on_batch_begin
+        else:
+            self.on_batch_begin = lambda batch, logs: None
+        if on_batch_end is not None:
+            self.on_batch_end = on_batch_end
+        else:
+            self.on_batch_end = lambda batch, logs: None
+        if on_train_begin is not None:
+            self.on_train_begin = on_train_begin
+        else:
+            self.on_train_begin = lambda logs: None
+        if on_train_end is not None:
+            self.on_train_end = on_train_end
+        else:
+            self.on_train_end = lambda logs: None
+from sys import stderr
+
+
+class TQDMCallback(Callback):
+    def __init__(self, outer_description="Training",
+                 inner_description_initial="Epoch: {epoch}",
+                 inner_description_update="Epoch: {epoch} - {metrics}",
+                 metric_format="{name}: {value:0.3f}",
+                 separator=", ",
+                 leave_inner=True,
+                 leave_outer=True,
+                 show_inner=True,
+                 show_outer=True,
+                 output_file=stderr,
+                 initial=0):
+        """
+        Construct a callback that will create and update progress bars.
+
+        :param outer_description: string for outer progress bar
+        :param inner_description_initial: initial format for epoch ("Epoch: {epoch}")
+        :param inner_description_update: format after metrics collected ("Epoch: {epoch} - {metrics}")
+        :param metric_format: format for each metric name/value pair ("{name}: {value:0.3f}")
+        :param separator: separator between metrics (", ")
+        :param leave_inner: True to leave inner bars
+        :param leave_outer: True to leave outer bars
+        :param show_inner: False to hide inner bars
+        :param show_outer: False to hide outer bar
+        :param output_file: output file (default sys.stderr)
+        :param initial: Initial counter state
+        """
+        self.outer_description = outer_description
+        self.inner_description_initial = inner_description_initial
+        self.inner_description_update = inner_description_update
+        self.metric_format = metric_format
+        self.separator = separator
+        self.leave_inner = leave_inner
+        self.leave_outer = leave_outer
+        self.show_inner = show_inner
+        self.show_outer = show_outer
+        self.output_file = output_file
+        self.tqdm_outer = None
+        self.tqdm_inner = None
+        self.epoch = None
+        self.running_logs = None
+        self.inner_count = None
+        self.initial = initial
+
+    def tqdm(self, desc, total, leave, initial=0):
+        """
+        Extension point. Override to provide custom options to tqdm initializer.
+        :param desc: Description string
+        :param total: Total number of updates
+        :param leave: Leave progress bar when done
+        :param initial: Initial counter state
+        :return: new progress bar
+        """
+        return tqdm(desc=desc, total=total, leave=leave, file=self.output_file, initial=initial)
+
+    def build_tqdm_outer(self, desc, total):
+        """
+        Extension point. Override to provide custom options to outer progress bars (Epoch loop)
+        :param desc: Description
+        :param total: Number of epochs
+        :return: new progress bar
+        """
+        return self.tqdm(desc=desc, total=total, leave=self.leave_outer, initial=self.initial)
+
+    def build_tqdm_inner(self, desc, total):
+        """
+        Extension point. Override to provide custom options to inner progress bars (Batch loop)
+        :param desc: Description
+        :param total: Number of batches
+        :return: new progress bar
+        """
+        return self.tqdm(desc=desc, total=total, leave=self.leave_inner)
+
+    def on_epoch_begin(self, epoch, logs={}):
+        self.epoch = epoch
+        desc = self.inner_description_initial.format(epoch=self.epoch)
+        self.mode = 0  # samples
+        if 'samples' in self.params:
+            self.inner_total = self.params['samples']
+        elif 'nb_sample' in self.params:
+            self.inner_total = self.params['nb_sample']
+        else:
+            self.mode = 1  # steps
+            self.inner_total = self.params['steps']
+        if self.show_inner:
+            self.tqdm_inner = self.build_tqdm_inner(desc=desc, total=self.inner_total)
+        self.inner_count = 0
+        self.running_logs = {}
+
+    def on_epoch_end(self, epoch, logs={}):
+        metrics = self.format_metrics(logs)
+        desc = self.inner_description_update.format(epoch=epoch, metrics=metrics)
+        if self.show_inner:
+            self.tqdm_inner.desc = desc
+            # set miniters and mininterval to 0 so last update displays
+            self.tqdm_inner.miniters = 0
+            self.tqdm_inner.mininterval = 0
+            self.tqdm_inner.update(self.inner_total - self.tqdm_inner.n)
+            self.tqdm_inner.close()
+        if self.show_outer:
+            self.tqdm_outer.update(1)
+
+    def on_batch_begin(self, batch, logs={}):
+        pass
+
+    def on_batch_end(self, batch, logs={}):
+        if self.mode == 0:
+            update = logs['size']
+        else:
+            update = 1
+        self.inner_count += update
+        if self.inner_count < self.inner_total:
+            self.append_logs(logs)
+            metrics = self.format_metrics(self.running_logs)
+            desc = self.inner_description_update.format(epoch=self.epoch, metrics=metrics)
+            if self.show_inner:
+                self.tqdm_inner.desc = desc
+                self.tqdm_inner.update(update)
+
+    def on_train_begin(self, logs={}):
+        if self.show_outer:
+            epochs = (self.params['epochs'] if 'epochs' in self.params
+                      else self.params['nb_epoch'])
+            self.tqdm_outer = self.build_tqdm_outer(desc=self.outer_description,
+                                                    total=epochs)
+
+    def on_train_end(self, logs={}):
+        if self.show_outer:
+            self.tqdm_outer.close()
+
+    def append_logs(self, logs):
+        metrics = self.params['metrics']
+        for metric, value in six.iteritems(logs):
+            if metric in metrics:
+                if metric in self.running_logs:
+                    self.running_logs[metric].append(value[()])
+                else:
+                    self.running_logs[metric] = [value[()]]
+
+    def format_metrics(self, logs):
+        metrics = self.params['metrics']
+        strings = [self.metric_format.format(name=metric, value=np.mean(logs[metric], axis=None)) for metric in metrics
+                   if
+                   metric in logs]
+        return self.separator.join(strings)

dataset.py

@@ -3,7 +3,7 @@ from torch.utils.data import Dataset
 from PIL import Image
 import os
 import json
-from build_vocab import Vocabulary, JsonReader
+from utils.build_vocab import Vocabulary, JsonReader
 import numpy as np
 from torchvision import transforms
 import pickle

logger.py

@@ -0,0 +1,71 @@
+# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514
+import tensorflow as tf
+import numpy as np
+import scipy.misc 
+try:
+    from StringIO import StringIO  # Python 2.7
+except ImportError:
+    from io import BytesIO         # Python 3.x
+
+
+class Logger(object):
+    
+    def __init__(self, log_dir):
+        """Create a summary writer logging to log_dir."""
+        self.writer = tf.summary.FileWriter(log_dir)
+
+    def scalar_summary(self, tag, value, step):
+        """Log a scalar variable."""
+        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
+        self.writer.add_summary(summary, step)
+
+    def image_summary(self, tag, images, step):
+        """Log a list of images."""
+
+        img_summaries = []
+        for i, img in enumerate(images):
+            # Write the image to a string
+            try:
+                s = StringIO()
+            except:
+                s = BytesIO()
+            scipy.misc.toimage(img).save(s, format="png")
+
+            # Create an Image object
+            img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
+                                       height=img.shape[0],
+                                       width=img.shape[1])
+            # Create a Summary value
+            img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))
+
+        # Create and write Summary
+        summary = tf.Summary(value=img_summaries)
+        self.writer.add_summary(summary, step)
+        
+    def histo_summary(self, tag, values, step, bins=1000):
+        """Log a histogram of the tensor of values."""
+
+        # Create a histogram using numpy
+        counts, bin_edges = np.histogram(values, bins=bins)
+
+        # Fill the fields of the histogram proto
+        hist = tf.HistogramProto()
+        hist.min = float(np.min(values))
+        hist.max = float(np.max(values))
+        hist.num = int(np.prod(values.shape))
+        hist.sum = float(np.sum(values))
+        hist.sum_squares = float(np.sum(values**2))
+
+        # Drop the start of the first bin
+        bin_edges = bin_edges[1:]
+
+        # Add bin edges and counts
+        for edge in bin_edges:
+            hist.bucket_limit.append(edge)
+        for c in counts:
+            hist.bucket.append(c)
+
+        # Create and write Summary
+        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
+        self.writer.add_summary(summary, step)
+        self.writer.flush()

models_debugger.py

@@ -0,0 +1,816 @@
+import torch
+import torch.nn as nn
+import torchvision
+import numpy as np
+from torch.autograd import Variable
+from torchvision.models.vgg import model_urls as vgg_model_urls
+import torchvision.models as models
+
+from utils.tcn import *
+
+
+class DenseNet121(nn.Module):
+    def __init__(self, classes=14, pretrained=True):
+        super(DenseNet121, self).__init__()
+        self.model = torchvision.models.densenet121(pretrained=pretrained)
+        num_in_features = self.model.classifier.in_features
+        self.model.classifier = nn.Sequential(
+            nn.Linear(in_features=num_in_features, out_features=classes, bias=True),
+            # nn.Sigmoid()
+        )
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.densenet121(x)
+        return x
+
+
+class DenseNet161(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(DenseNet161, self).__init__()
+        self.model = torchvision.models.densenet161(pretrained=pretrained)
+        num_in_features = self.model.classifier.in_features
+        self.model.classifier = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class DenseNet169(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(DenseNet169, self).__init__()
+        self.model = torchvision.models.densenet169(pretrained=pretrained)
+        num_in_features = self.model.classifier.in_features
+        self.model.classifier = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class DenseNet201(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(DenseNet201, self).__init__()
+        self.model = torchvision.models.densenet201(pretrained=pretrained)
+        num_in_features = self.model.classifier.in_features
+        self.model.classifier = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class ResNet18(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(ResNet18, self).__init__()
+        self.model = torchvision.models.resnet18(pretrained=pretrained)
+        num_in_features = self.model.fc.in_features
+        self.model.fc = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class ResNet34(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(ResNet34, self).__init__()
+        self.model = torchvision.models.resnet34(pretrained=pretrained)
+        num_in_features = self.model.fc.in_features
+        self.model.fc = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class ResNet50(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(ResNet50, self).__init__()
+        self.model = torchvision.models.resnet50(pretrained=pretrained)
+        num_in_features = self.model.fc.in_features
+        self.model.fc = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class ResNet101(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(ResNet101, self).__init__()
+        self.model = torchvision.models.resnet101(pretrained=pretrained)
+        num_in_features = self.model.fc.in_features
+        self.model.fc = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class ResNet152(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(ResNet152, self).__init__()
+        self.model = torchvision.models.resnet152(pretrained=pretrained)
+        num_in_features = self.model.fc.in_features
+        self.model.fc = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class VGG19(nn.Module):
+    def __init__(self, classes=14, pretrained=True):
+        super(VGG19, self).__init__()
+        self.model = torchvision.models.vgg19_bn(pretrained=pretrained)
+        self.model.classifier = nn.Sequential(
+            self.__init_linear(in_features=25088, out_features=4096),
+            nn.ReLU(),
+            nn.Dropout(0.5),
+            self.__init_linear(in_features=4096, out_features=4096),
+            nn.ReLU(),
+            nn.Dropout(0.5),
+            self.__init_linear(in_features=4096, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class VGG(nn.Module):
+    def __init__(self, tags_num):
+        super(VGG, self).__init__()
+        vgg_model_urls['vgg19'] = vgg_model_urls['vgg19'].replace('https://', 'http://')
+        self.vgg19 = models.vgg19(pretrained=True)
+        vgg19_classifier = list(self.vgg19.classifier.children())[:-1]
+        self.classifier = nn.Sequential(*vgg19_classifier)
+        self.fc = nn.Linear(4096, tags_num)
+        self.fc.apply(self.init_weights)
+        self.bn = nn.BatchNorm1d(tags_num, momentum=0.1)
+#        self.init_weights()
+
+    def init_weights(self, m):
+        if type(m) == nn.Linear:
+            self.fc.weight.data.normal_(0, 0.1)
+            self.fc.bias.data.fill_(0)
+
+    def forward(self, images) -> object:
+        """
+
+        :rtype: object
+        """
+        visual_feats = self.vgg19.features(images)
+        tags_classifier = visual_feats.view(visual_feats.size(0), -1)
+        tags_classifier = self.bn(self.fc(self.classifier(tags_classifier)))
+        return tags_classifier
+
+
+class InceptionV3(nn.Module):
+    def __init__(self, classes=156, pretrained=True):
+        super(InceptionV3, self).__init__()
+        self.model = torchvision.models.inception_v3(pretrained=pretrained)
+        num_in_features = self.model.classifier.in_features
+        self.model.classifier = nn.Sequential(
+            self.__init_linear(in_features=num_in_features, out_features=classes),
+            # nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.model(x)
+        return x
+
+
+class CheXNetDenseNet121(nn.Module):
+    def __init__(self, classes=14, pretrained=True):
+        super(CheXNetDenseNet121, self).__init__()
+        self.densenet121 = torchvision.models.densenet121(pretrained=pretrained)
+        num_in_features = self.densenet121.classifier.in_features
+        self.densenet121.classifier = nn.Sequential(
+            nn.Linear(in_features=num_in_features, out_features=classes, bias=True),
+            nn.Sigmoid()
+        )
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.densenet121(x)
+        return x
+
+
+class CheXNet(nn.Module):
+    def __init__(self, classes=156):
+        super(CheXNet, self).__init__()
+        self.densenet121 = CheXNetDenseNet121(classes=14)
+        self.densenet121 = torch.nn.DataParallel(self.densenet121).cuda()
+        self.densenet121.load_state_dict(torch.load('./models/CheXNet.pth.tar')['state_dict'])
+        self.densenet121.module.densenet121.classifier = nn.Sequential(
+            self.__init_linear(1024, classes),
+            nn.Sigmoid()
+        )
+
+    def __init_linear(self, in_features, out_features):
+        func = nn.Linear(in_features=in_features, out_features=out_features, bias=True)
+        func.weight.data.normal_(0, 0.1)
+        return func
+
+    def forward(self, x) -> object:
+        """
+
+        :rtype: object
+        """
+        x = self.densenet121(x)
+        return x
+
+
+class ModelFactory(object):
+    def __init__(self, model_name, pretrained, classes):
+        self.model_name = model_name
+        self.pretrained = pretrained
+        self.classes = classes
+
+    def create_model(self):
+        if self.model_name == 'VGG19':
+            _model = VGG19(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'DenseNet121':
+            _model = DenseNet121(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'DenseNet161':
+            _model = DenseNet161(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'DenseNet169':
+            _model = DenseNet169(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'DenseNet201':
+            _model = DenseNet201(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'CheXNet':
+            _model = CheXNet(classes=self.classes)
+        elif self.model_name == 'ResNet18':
+            _model = ResNet18(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'ResNet34':
+            _model = ResNet34(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'ResNet50':
+            _model = ResNet50(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'ResNet101':
+            _model = ResNet101(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'ResNet152':
+            _model = ResNet152(pretrained=self.pretrained, classes=self.classes)
+        elif self.model_name == 'VGG':
+            _model = VGG(tags_num=self.classes)
+        else:
+            _model = CheXNet(classes=self.classes)
+
+        return _model
+
+
+class EncoderCNN(nn.Module):
+    def __init__(self, embed_size, pretrained=True):
+        super(EncoderCNN, self).__init__()
+        # TODO Extract Image features from CNN based on other models
+        resnet = models.resnet152(pretrained=pretrained)
+        modules = list(resnet.children())[:-1]
+        self.resnet = nn.Sequential(*modules)
+        self.linear = nn.Linear(resnet.fc.in_features, embed_size)
+        self.bn = nn.BatchNorm1d(embed_size, momentum=0.1)
+        self.__init_weights()
+
+    def __init_weights(self):
+        self.linear.weight.data.normal_(0.0, 0.1)
+        self.linear.bias.data.fill_(0)
+
+    def forward(self, images) -> object:
+        """
+
+        :rtype: object
+        """
+        features = self.resnet(images)
+        features = Variable(features.data)
+        features = features.view(features.size(0), -1)
+        features = self.bn(self.linear(features))
+        return features
+
+
+class DecoderRNN(nn.Module):
+    def __init__(self, embed_size, hidden_size, vocab_size, num_layers, n_max=50):
+        super(DecoderRNN, self).__init__()
+        self.embed = nn.Embedding(vocab_size, embed_size)
+        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
+        self.linear = nn.Linear(hidden_size, vocab_size)
+        self.__init_weights()
+        self.n_max = n_max
+
+    def __init_weights(self):
+        self.embed.weight.data.uniform_(-0.1, 0.1)
+        self.linear.weight.data.uniform_(-0.1, 0.1)
+        self.linear.bias.data.fill_(0)
+
+    def forward(self, features, captions) -> object:
+        """
+
+        :rtype: object
+        """
+        embeddings = self.embed(captions)
+        embeddings = torch.cat((features.unsqueeze(1), embeddings), 1)
+        hidden, _ = self.lstm(embeddings)
+        outputs = self.linear(hidden[:, -1, :])
+        return outputs
+
+    def sample(self, features, start_tokens):
+        sampled_ids = np.zeros((np.shape(features)[0], self.n_max))
+        predicted = start_tokens
+        embeddings = features
+        embeddings = embeddings.unsqueeze(1)
+
+        for i in range(self.n_max):
+            predicted = self.embed(predicted)
+            embeddings = torch.cat([embeddings, predicted], dim=1)
+            hidden_states, _ = self.lstm(embeddings)
+            hidden_states = hidden_states[:, -1, :]
+            outputs = self.linear(hidden_states)
+            predicted = torch.max(outputs, 1)[1]
+            sampled_ids[:, i] = predicted
+            predicted = predicted.unsqueeze(1)
+        return sampled_ids
+
+
+class VisualFeatureExtractor(nn.Module):
+    def __init__(self, pretrained=False):
+        super(VisualFeatureExtractor, self).__init__()
+        resnet = models.resnet152(pretrained=pretrained)
+        modules = list(resnet.children())[:-1]
+        self.resnet = nn.Sequential(*modules)
+        self.out_features = resnet.fc.in_features
+
+    def forward(self, images) -> object:
+        """
+
+        :rtype: object
+        """
+        features = self.resnet(images)
+        features = features.view(features.size(0), -1)
+        return features
+
+
+class MLC(nn.Module):
+    def __init__(self, classes=156, sementic_features_dim=512, fc_in_features=2048, k=10):
+        super(MLC, self).__init__()
+        self.classifier = nn.Linear(in_features=fc_in_features, out_features=classes)
+        self.embed = nn.Embedding(classes, sementic_features_dim)
+        self.k = k
+        self.softmax = nn.Softmax()
+
+    def forward(self, visual_features) -> object:
+        """
+
+        :rtype: object
+        """
+        tags = self.softmax(self.classifier(visual_features))
+        semantic_features = self.embed(torch.topk(tags, self.k)[1])
+        return tags, semantic_features
+
+
+class CoAttention(nn.Module):
+    def __init__(self, embed_size=512, hidden_size=512, visual_size=2048):
+        super(CoAttention, self).__init__()
+        self.W_v = nn.Linear(in_features=visual_size, out_features=visual_size)
+        self.bn_v = nn.BatchNorm1d(num_features=visual_size, momentum=0.1)
+
+        self.W_v_h = nn.Linear(in_features=hidden_size, out_features=visual_size)
+        self.bn_v_h = nn.BatchNorm1d(num_features=visual_size, momentum=0.1)
+
+        self.W_v_att = nn.Linear(in_features=visual_size, out_features=visual_size)
+        self.bn_v_att = nn.BatchNorm1d(num_features=visual_size, momentum=0.1)
+
+        self.W_a = nn.Linear(in_features=hidden_size, out_features=hidden_size)
+        self.bn_a = nn.BatchNorm1d(num_features=10, momentum=0.1)
+
+        self.W_a_h = nn.Linear(in_features=hidden_size, out_features=hidden_size)
+        self.bn_a_h = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_a_att = nn.Linear(in_features=hidden_size, out_features=hidden_size, bias=True)
+        self.bn_a_att = nn.BatchNorm1d(num_features=10, momentum=0.1)
+
+        self.W_fc = nn.Linear(in_features=visual_size + hidden_size, out_features=embed_size)
+        self.bn_fc = nn.BatchNorm1d(num_features=embed_size, momentum=0.1)
+
+        self.tanh = nn.Tanh()
+        self.softmax = nn.Softmax()
+
+    def forward(self, visual_features, semantic_features, h_sent) -> object:
+        """
+        only training
+        :rtype: object
+        """
+        W_v = self.bn_v(self.W_v(visual_features))
+        W_v_h = self.bn_v_h(self.W_v_h(h_sent.squeeze(1)))
+
+        alpha_v = self.softmax(self.bn_v_att(self.W_v_att(self.tanh(W_v + W_v_h))))
+        v_att = torch.mul(alpha_v, visual_features)
+        # v_att = torch.mul(alpha_v, visual_features).sum(1).unsqueeze(1)
+
+        W_a_h = self.bn_a_h(self.W_a_h(h_sent))
+        W_a = self.bn_a(self.W_a(semantic_features))
+        alpha_a = self.softmax(self.bn_a_att(self.W_a_att(self.tanh(torch.add(W_a_h, W_a)))))
+        a_att = torch.mul(alpha_a, semantic_features).sum(1)
+        # a_att = (alpha_a * semantic_features).sum(1)
+        ctx = self.bn_fc(self.W_fc(torch.cat([v_att, a_att], dim=1)))
+        # return self.W_fc(self.bn_fc(torch.cat([v_att, a_att], dim=1)))
+        return ctx, v_att
+
+
+class SentenceLSTM(nn.Module):
+    def __init__(self, embed_size=512, hidden_size=512, num_layers=1):
+        super(SentenceLSTM, self).__init__()
+        self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hidden_size, num_layers=num_layers)
+        self.W_t_h = nn.Linear(in_features=hidden_size, out_features=embed_size, bias=True)
+        self.bn_t_h = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_t_ctx = nn.Linear(in_features=embed_size, out_features=embed_size, bias=True)
+        self.bn_t_ctx = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_stop_s_1 = nn.Linear(in_features=hidden_size, out_features=embed_size, bias=True)
+        self.bn_stop_s_1 = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_stop_s = nn.Linear(in_features=hidden_size, out_features=embed_size, bias=True)
+        self.bn_stop_s = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_stop = nn.Linear(in_features=embed_size, out_features=2, bias=True)
+        self.bn_stop = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_topic = nn.Linear(in_features=embed_size, out_features=embed_size, bias=True)
+        self.bn_topic = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.W_topic_2 = nn.Linear(in_features=embed_size, out_features=embed_size, bias=True)
+        self.bn_topic_2 = nn.BatchNorm1d(num_features=1, momentum=0.1)
+
+        self.sigmoid = nn.Sigmoid()
+        self.tanh = nn.Tanh()
+
+    # def forward(self, ctx, prev_hidden_state, states=None) -> object:
+    #     """
+    #     Only training
+    #     :rtype: object
+    #     """
+    #     ctx = ctx.unsqueeze(1)
+    #     hidden_state, states = self.lstm(ctx, states)
+    #     topic = self.bn_topic(self.W_topic(self.sigmoid(self.bn_t_h(self.W_t_h(hidden_state))
+    #                                                     + self.bn_t_ctx(self.W_t_ctx(ctx)))))
+    #     p_stop = self.bn_stop(self.W_stop(self.sigmoid(self.bn_stop_s_1(self.W_stop_s_1(prev_hidden_state))
+    #                                       + self.bn_stop_s(self.W_stop_s(hidden_state)))))
+    #     return topic, p_stop, hidden_state, states
+
+    def forward(self, ctx, prev_hidden_state, states=None) -> object:
+        """
+        v2
+        :rtype: object
+        """
+        ctx = ctx.unsqueeze(1)
+        hidden_state, states = self.lstm(ctx, states)
+        topic = self.bn_topic(self.W_topic(self.tanh(self.bn_t_h(self.W_t_h(hidden_state)
+                                                                 + self.W_t_ctx(ctx)))))
+        p_stop = self.bn_stop(self.W_stop(self.tanh(self.bn_stop_s(self.W_stop_s_1(prev_hidden_state)
+                                                                   + self.W_stop_s(hidden_state)))))
+        return topic, p_stop, hidden_state, states
+
+
+class SentenceTCN(nn.Module):
+    def __init__(self,
+                 input_channel=10,
+                 embed_size=512,
+                 output_size=512,
+                 nhid=512,
+                 levels=8,
+                 kernel_size=2,
+                 dropout=0):
+        super(SentenceTCN, self).__init__()
+        channel_sizes = [nhid] * levels
+        self.tcn = TCN(input_size=input_channel,
+                       output_size=output_size,
+                       num_channels=channel_sizes,
+                       kernel_size=kernel_size,
+                       dropout=dropout)
+        self.W_t_h = nn.Linear(in_features=output_size, out_features=embed_size, bias=True)
+        self.W_t_ctx = nn.Linear(in_features=output_size, out_features=embed_size, bias=True)
+        self.W_stop_s_1 = nn.Linear(in_features=output_size, out_features=embed_size, bias=True)
+        self.W_stop_s = nn.Linear(in_features=output_size, out_features=embed_size, bias=True)
+        self.W_stop = nn.Linear(in_features=embed_size, out_features=2, bias=True)
+        self.t_w = nn.Linear(in_features=5120, out_features=2, bias=True)
+        self.tanh = nn.Tanh()
+
+    def forward(self, ctx, prev_output) -> object:
+        """
+
+        :rtype: object
+        """
+        output = self.tcn.forward(ctx)
+        topic = self.tanh(self.W_t_h(output) + self.W_t_ctx(ctx[:, -1, :]).squeeze(1))
+        p_stop = self.W_stop(self.tanh(self.W_stop_s_1(prev_output) + self.W_stop_s(output)))
+        return topic, p_stop, output
+
+
+class WordLSTM(nn.Module):
+    def __init__(self, embed_size, hidden_size, vocab_size, num_layers, n_max=50):
+        super(WordLSTM, self).__init__()
+        self.embed = nn.Embedding(vocab_size, embed_size)
+        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
+        self.linear = nn.Linear(hidden_size, vocab_size)
+        self.__init_weights()
+        self.n_max = n_max
+        self.vocab_size = vocab_size
+
+    def __init_weights(self):
+        self.embed.weight.data.uniform_(-0.1, 0.1)
+        self.linear.weight.data.uniform_(-0.1, 0.1)
+        self.linear.bias.data.fill_(0)
+
+    def forward(self, topic_vec, captions) -> object:
+        """
+
+        :rtype: object
+        """
+        embeddings = self.embed(captions)
+        embeddings = torch.cat((topic_vec, embeddings), 1)
+        hidden, _ = self.lstm(embeddings)
+        outputs = self.linear(hidden[:, -1, :])
+        return outputs
+
+    def val(self, features, start_tokens):
+        samples = torch.zeros((np.shape(features)[0], self.n_max, self.vocab_size))
+        samples[:, 0, start_tokens[0]] = 1
+        predicted = start_tokens
+        embeddings = features
+        embeddings = embeddings
+
+        for i in range(1, self.n_max):
+            predicted = self.embed(predicted)
+            embeddings = torch.cat([embeddings, predicted], dim=1)
+            hidden_states, _ = self.lstm(embeddings)
+            hidden_states = hidden_states[:, -1, :]
+            outputs = self.linear(hidden_states)
+            samples[:, i, :] = outputs
+            predicted = torch.max(outputs, 1)[1]
+            predicted = predicted.unsqueeze(1)
+        return samples
+
+    def sample(self, features, start_tokens):
+        sampled_ids = np.zeros((np.shape(features)[0], self.n_max))
+        sampled_ids[:, 0] = start_tokens.view(-1,)
+        predicted = start_tokens
+        embeddings = features
+        embeddings = embeddings
+
+        for i in range(1, self.n_max):
+            predicted = self.embed(predicted)
+            embeddings = torch.cat([embeddings, predicted], dim=1)
+            hidden_states, _ = self.lstm(embeddings)
+            hidden_states = hidden_states[:, -1, :]
+            outputs = self.linear(hidden_states)
+            predicted = torch.max(outputs, 1)[1]
+            sampled_ids[:, i] = predicted
+            predicted = predicted.unsqueeze(1)
+        return sampled_ids
+
+
+class WordTCN(nn.Module):
+    def __init__(self,
+                 input_channel=11,
+                 vocab_size=1000,
+                 embed_size=512,
+                 output_size=512,
+                 nhid=512,
+                 levels=8,
+                 kernel_size=2,
+                 dropout=0,
+                 n_max=50):
+        super(WordTCN, self).__init__()
+        self.vocab_size = vocab_size
+        self.embed_size = embed_size
+        self.output_size = output_size
+        channel_sizes = [nhid] * levels
+        self.kernel_size = kernel_size
+        self.dropout = dropout
+        self.n_max = n_max
+        self.embed = nn.Embedding(vocab_size, embed_size)
+        self.W_out = nn.Linear(in_features=output_size, out_features=vocab_size, bias=True)
+        self.tcn = TCN(input_size=input_channel,
+                       output_size=output_size,
+                       num_channels=channel_sizes,
+                       kernel_size=kernel_size,
+                       dropout=dropout)
+
+    def forward(self, topic_vec, captions) -> object:
+        """
+
+        :rtype: object
+        """
+        captions = self.embed(captions)
+        embeddings = torch.cat([topic_vec, captions], dim=1)
+        output = self.tcn.forward(embeddings)
+        words = self.W_out(output)
+        return words
+
+
+if __name__ == '__main__':
+    import warnings
+    warnings.filterwarnings("ignore")
+    images = torch.randn((4, 3, 224, 224))
+    captions = torch.ones((4, 10)).long()
+    hidden_state = torch.randn((4, 1, 512))
+
+    print("images:{}".format(images.shape))
+    print("captions:{}".format(captions.shape))
+    print("hidden_states:{}".format(hidden_state.shape))
+
+    extractor = VisualFeatureExtractor()
+    visual_features = extractor.forward(images)
+    print("visual_features:{}".format(visual_features.shape))
+
+    mlc = MLC()
+    tags, semantic_features = mlc.forward(visual_features)
+    print("tags:{}".format(tags.shape))
+    print("semantic_features:{}".format(semantic_features.shape))
+
+    co_att = CoAttention()
+    ctx, v_att = co_att.forward(visual_features, semantic_features, hidden_state)
+    print("ctx:{}".format(ctx.shape))
+    print("v_att:{}".format(v_att.shape))
+
+    sent_lstm = SentenceLSTM()
+    topic, p_stop, hidden_state, states = sent_lstm.forward(ctx, hidden_state)
+    print("Topic:{}".format(topic.shape))
+    print("P_STOP:{}".format(p_stop.shape))
+
+    word_lstm = WordLSTM(embed_size=512, hidden_size=512, vocab_size=100, num_layers=1)
+    words = word_lstm.forward(topic, captions)
+    print("words:{}".format(words.shape))
+
+    # Expected Output
+    # images: torch.Size([4, 3, 224, 224])
+    # captions: torch.Size([4, 1, 10])
+    # hidden_states: torch.Size([4, 1, 512])
+    # visual_features: torch.Size([4, 2048, 7, 7])
+    # tags: torch.Size([4, 156])
+    # semantic_features: torch.Size([4, 10, 512])
+    # ctx: torch.Size([4, 512])
+    # Topic: torch.Size([4, 1, 512])
+    # P_STOP: torch.Size([4, 1, 2])
+    # words: torch.Size([4, 1000])
+
+    # images = torch.randn((4, 3, 224, 224))
+    # captions = torch.ones((4, 3, 10)).long()
+    # prev_outputs = torch.randn((4, 512))
+    # now_words = torch.ones((4, 1))
+    #
+    # ctx_records = torch.zeros((4, 10, 512))
+    # captions = torch.zeros((4, 10)).long()
+    #
+    # print("images:{}".format(images.shape))
+    # print("captions:{}".format(captions.shape))
+    # print("hidden_states:{}".format(prev_outputs.shape))
+    #
+    # extractor = VisualFeatureExtractor()
+    # visual_features = extractor.forward(images)
+    # print("visual_features:{}".format(visual_features.shape))
+    #
+    # mlc = MLC()
+    # tags, semantic_features = mlc.forward(visual_features)
+    # print("tags:{}".format(tags.shape))
+    # print("semantic_features:{}".format(semantic_features.shape))
+    #
+    # co_att = CoAttention()
+    # ctx = co_att.forward(visual_features, semantic_features, prev_outputs)
+    # print("ctx:{}".format(ctx.shape))
+    #
+    # ctx_records[:, 0, :] = ctx
+    #
+    # sent_tcn = SentenceTCN()
+    # topic, p_stop, prev_outputs = sent_tcn.forward(ctx_records, prev_outputs)
+    # print("Topic:{}".format(topic.shape))
+    # print("P_STOP:{}".format(p_stop.shape))
+    # print("Prev_Outputs:{}".format(prev_outputs.shape))
+    #
+    # captions[:, 0] = now_words.view(-1,)
+    #
+    # word_tcn = WordTCN()
+    # words = word_tcn.forward(topic, captions)
+    # print("words:{}".format(words.shape))
+

tcn.py

@@ -0,0 +1,83 @@
+import torch
+import torch.nn as nn
+from torch.nn.utils import weight_norm
+
+
+class Chomp1d(nn.Module):
+    def __init__(self, chomp_size):
+        super(Chomp1d, self).__init__()
+        self.chomp_size = chomp_size
+
+    def forward(self, x) -> object:
+        return x[:, :, :-self.chomp_size].contiguous()
+
+
+class TemporalBlock(nn.Module):
+    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):
+        super(TemporalBlock, self).__init__()
+        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,
+                                           stride=stride, padding=padding, dilation=dilation))
+        self.chomp1 = Chomp1d(padding)
+        self.relu1 = nn.ReLU(inplace=False)
+        self.dropout1 = nn.Dropout(dropout)
+
+        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,
+                                           stride=stride, padding=padding, dilation=dilation))
+        self.chomp2 = Chomp1d(padding)
+        self.relu2 = nn.ReLU(inplace=False)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,
+                                 self.conv2, self.chomp2, self.relu2, self.dropout2)
+        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None
+        self.relu = nn.ReLU(inplace=False)
+        self.init_weights()
+
+    def init_weights(self):
+        self.conv1.weight.data.normal_(0, 0.01)
+        self.conv2.weight.data.normal_(0, 0.01)
+        if self.downsample is not None:
+            self.downsample.weight.data.normal_(0, 0.01)
+
+    def forward(self, x) -> object:
+        out = self.net(x)
+        res = x if self.downsample is None else self.downsample(x)
+        return self.relu(out + res)
+
+
+class TemporalConvNet(nn.Module):
+    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
+        super(TemporalConvNet, self).__init__()
+        layers = []
+        num_levels = len(num_channels)
+        for i in range(num_levels):
+            dilation_size = 2 ** i
+            in_channels = num_inputs if i == 0 else num_channels[i-1]
+            out_channels = num_channels[i]
+            layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,
+                                     padding=(kernel_size-1) * dilation_size, dropout=dropout)]
+
+        self.network = nn.Sequential(*layers)
+
+    def forward(self, x) -> object:
+        return self.network(x)
+
+
+class TCN(nn.Module):
+    def __init__(self, input_size, output_size, num_channels, kernel_size=2, dropout=0):
+        super(TCN, self).__init__()
+        self.tcn = TemporalConvNet(num_inputs=input_size,
+                                   num_channels=num_channels,
+                                   kernel_size=kernel_size,
+                                   dropout=dropout)
+        self.linear = nn.Linear(num_channels[-1], output_size)
+        self.init_weights()
+
+    def init_weights(self):
+        self.linear.weight.data.normal_(0, 0.01)
+        self.linear.bias.data.fill_(0)
+
+    def forward(self, inputs) -> object:
+        y = self.tcn.forward(inputs)
+        output = self.linear(y[:, :, -1])
+        return output