<p dir="auto">import numpy as np<br>
from keras.preprocessing.text import Tokenizer<br>
from keras.preprocessing.sequence import pad_sequences<br>
from keras.utils import to_categorical<br>
from keras.models import Sequential<br>
from keras.layers import Dense, Embedding, LSTM<br>
from keras import optimizers<br>
from keras.models import load_model<br>
import json, argparse, os<br>
import re<br>
import io<br>
import sys</p>
<h1 dir="auto">Path to training and testing data file. This data can be downloaded from a link, details of which will be provided.</h1>
<p dir="auto">trainDataPath = "train.txt"<br>
testDataPath = "dev.txt"</p>
<h1 dir="auto">Output file that will be generated. This file can be directly submitted.</h1>
<p dir="auto">solutionPath = "test.txt"</p>
<h1 dir="auto">Path to directory where GloVe file is saved.</h1>
<p dir="auto">gloveDir = "./"</p>
<p dir="auto">NUM_FOLDS = 1                   # Value of K in K-fold Cross Validation<br>
NUM_CLASSES = 4                 # Number of classes - Happy, Sad, Angry, Others<br>
MAX_NB_WORDS = 20000                # To set the upper limit on the number of tokens extracted using keras.preprocessing.text.Tokenizer<br>
MAX_SEQUENCE_LENGTH = 100         # All sentences having lesser number of words than this will be padded<br>
EMBEDDING_DIM = 100               # The dimension of the word embeddings<br>
BATCH_SIZE = 200                  # The batch size to be chosen for training the model.<br>
LSTM_DIM = 128                    # The dimension of the representations learnt by the LSTM model<br>
DROPOUT = 0.2                     # Fraction of the units to drop for the linear transformation of the inputs. Ref - <a href="https://keras.io/layers/recurrent/" rel="nofollow">https://keras.io/layers/recurrent/</a><br>
NUM_EPOCHS = 15                  # Number of epochs to train a model for</p>
<p dir="auto">label2emotion = {0:"others", 1:"happy", 2: "sad", 3:"angry"}<br>
emotion2label = {"others":0, "happy":1, "sad":2, "angry":3}</p>
<p dir="auto">def preprocessData(dataFilePath, mode):<br>
"""Load data from a file, process and return indices, conversations and labels in separate lists<br>
Input:<br>
dataFilePath : Path to train/test file to be processed<br>
mode : "train" mode returns labels. "test" mode doesn't return labels.<br>
Output:<br>
indices : Unique conversation ID list<br>
conversations : List of 3 turn conversations, processed and each turn separated by the  tag<br>
labels : [Only available in "train" mode] List of labels<br>
"""<br>
indices = []<br>
conversations = []<br>
labels = []<br>
with io.open(dataFilePath, encoding="utf8") as finput:<br>
finput.readline()<br>
for line in finput:<br>
# Convert multiple instances of . ? ! , to single instance<br>
# okay...sure -&gt; okay . sure<br>
# okay???sure -&gt; okay ? sure<br>
# Add whitespace around such punctuation<br>
# okay!sure -&gt; okay ! sure<br>
repeatedChars = ['.', '?', '!', ',']<br>
for c in repeatedChars:<br>
lineSplit = line.split(c)<br>
while True:<br>
try:<br>
lineSplit.remove('')<br>
except:<br>
break<br>
cSpace = ' ' + c + ' '<br>
line = cSpace.join(lineSplit)</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="        line = line.strip().split('\t')
        if mode == &quot;train&quot;:
            # Train data contains id, 3 turns and label
            label = emotion2label[line[4]]
            labels.append(label)
        
        conv = ' &lt;eos&gt; '.join(line[1:4])
        
        # Remove any duplicate spaces
        duplicateSpacePattern = re.compile(r'\ +')
        conv = re.sub(duplicateSpacePattern, ' ', conv)
        
        indices.append(int(line[0]))
        conversations.append(conv.lower())

if mode == &quot;train&quot;:
    return indices, conversations, labels
else:
    return indices, conversations"><pre class="notranslate"><code class="notranslate">        line = line.strip().split('\t')
        if mode == "train":
            # Train data contains id, 3 turns and label
            label = emotion2label[line[4]]
            labels.append(label)
        
        conv = ' &lt;eos&gt; '.join(line[1:4])
        
        # Remove any duplicate spaces
        duplicateSpacePattern = re.compile(r'\ +')
        conv = re.sub(duplicateSpacePattern, ' ', conv)
        
        indices.append(int(line[0]))
        conversations.append(conv.lower())

if mode == "train":
    return indices, conversations, labels
else:
    return indices, conversations
</code></pre></div>
<p dir="auto">def getMetrics(predictions, ground):<br>
"""Given predicted labels and the respective ground truth labels, display some metrics<br>
Input: shape [# of samples, NUM_CLASSES]<br>
predictions : Model output. Every row has 4 decimal values, with the highest belonging to the predicted class<br>
ground : Ground truth labels, converted to one-hot encodings. A sample belonging to Happy class will be [0, 1, 0, 0]<br>
Output:<br>
accuracy : Average accuracy<br>
microPrecision : Precision calculated on a micro level. Ref - <a href="https://datascience.stackexchange.com/questions/15989/micro-average-vs-macro-average-performance-in-a-multiclass-classification-settin/16001" rel="nofollow">https://datascience.stackexchange.com/questions/15989/micro-average-vs-macro-average-performance-in-a-multiclass-classification-settin/16001</a><br>
microRecall : Recall calculated on a micro level<br>
microF1 : Harmonic mean of microPrecision and microRecall. Higher value implies better classification<br>
"""<br>
# [0.1, 0.3 , 0.2, 0.1] -&gt; [0, 1, 0, 0]<br>
discretePredictions = to_categorical(predictions.argmax(axis=1))</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="truePositives = np.sum(discretePredictions*ground, axis=0)
falsePositives = np.sum(np.clip(discretePredictions - ground, 0, 1), axis=0)
falseNegatives = np.sum(np.clip(ground-discretePredictions, 0, 1), axis=0)

print(&quot;True Positives per class : &quot;, truePositives)
print(&quot;False Positives per class : &quot;, falsePositives)
print(&quot;False Negatives per class : &quot;, falseNegatives)

# ------------- Macro level calculation ---------------
macroPrecision = 0
macroRecall = 0
# We ignore the &quot;Others&quot; class during the calculation of Precision, Recall and F1
for c in range(1, NUM_CLASSES):
    precision = truePositives[c] / (truePositives[c] + falsePositives[c])
    macroPrecision += precision
    recall = truePositives[c] / (truePositives[c] + falseNegatives[c])
    macroRecall += recall
    f1 = ( 2 * recall * precision ) / (precision + recall) if (precision+recall) &gt; 0 else 0
    print(&quot;Class %s : Precision : %.3f, Recall : %.3f, F1 : %.3f&quot; % (label2emotion[c], precision, recall, f1))

macroPrecision /= 3
macroRecall /= 3
macroF1 = (2 * macroRecall * macroPrecision ) / (macroPrecision + macroRecall) if (macroPrecision+macroRecall) &gt; 0 else 0
print(&quot;Ignoring the Others class, Macro Precision : %.4f, Macro Recall : %.4f, Macro F1 : %.4f&quot; % (macroPrecision, macroRecall, macroF1))   

# ------------- Micro level calculation ---------------
truePositives = truePositives[1:].sum()
falsePositives = falsePositives[1:].sum()
falseNegatives = falseNegatives[1:].sum()    

print(&quot;Ignoring the Others class, Micro TP : %d, FP : %d, FN : %d&quot; % (truePositives, falsePositives, falseNegatives))

microPrecision = truePositives / (truePositives + falsePositives)
microRecall = truePositives / (truePositives + falseNegatives)

microF1 = ( 2 * microRecall * microPrecision ) / (microPrecision + microRecall) if (microPrecision+microRecall) &gt; 0 else 0
# -----------------------------------------------------

predictions = predictions.argmax(axis=1)
ground = ground.argmax(axis=1)
accuracy = np.mean(predictions==ground)

print(&quot;Accuracy : %.4f, Micro Precision : %.4f, Micro Recall : %.4f, Micro F1 : %.4f&quot; % (accuracy, microPrecision, microRecall, microF1))
return accuracy, microPrecision, microRecall, microF1"><pre class="notranslate"><code class="notranslate">truePositives = np.sum(discretePredictions*ground, axis=0)
falsePositives = np.sum(np.clip(discretePredictions - ground, 0, 1), axis=0)
falseNegatives = np.sum(np.clip(ground-discretePredictions, 0, 1), axis=0)

print("True Positives per class : ", truePositives)
print("False Positives per class : ", falsePositives)
print("False Negatives per class : ", falseNegatives)

# ------------- Macro level calculation ---------------
macroPrecision = 0
macroRecall = 0
# We ignore the "Others" class during the calculation of Precision, Recall and F1
for c in range(1, NUM_CLASSES):
    precision = truePositives[c] / (truePositives[c] + falsePositives[c])
    macroPrecision += precision
    recall = truePositives[c] / (truePositives[c] + falseNegatives[c])
    macroRecall += recall
    f1 = ( 2 * recall * precision ) / (precision + recall) if (precision+recall) &gt; 0 else 0
    print("Class %s : Precision : %.3f, Recall : %.3f, F1 : %.3f" % (label2emotion[c], precision, recall, f1))

macroPrecision /= 3
macroRecall /= 3
macroF1 = (2 * macroRecall * macroPrecision ) / (macroPrecision + macroRecall) if (macroPrecision+macroRecall) &gt; 0 else 0
print("Ignoring the Others class, Macro Precision : %.4f, Macro Recall : %.4f, Macro F1 : %.4f" % (macroPrecision, macroRecall, macroF1))   

# ------------- Micro level calculation ---------------
truePositives = truePositives[1:].sum()
falsePositives = falsePositives[1:].sum()
falseNegatives = falseNegatives[1:].sum()    

print("Ignoring the Others class, Micro TP : %d, FP : %d, FN : %d" % (truePositives, falsePositives, falseNegatives))

microPrecision = truePositives / (truePositives + falsePositives)
microRecall = truePositives / (truePositives + falseNegatives)

microF1 = ( 2 * microRecall * microPrecision ) / (microPrecision + microRecall) if (microPrecision+microRecall) &gt; 0 else 0
# -----------------------------------------------------

predictions = predictions.argmax(axis=1)
ground = ground.argmax(axis=1)
accuracy = np.mean(predictions==ground)

print("Accuracy : %.4f, Micro Precision : %.4f, Micro Recall : %.4f, Micro F1 : %.4f" % (accuracy, microPrecision, microRecall, microF1))
return accuracy, microPrecision, microRecall, microF1
</code></pre></div>
<p dir="auto">def writeNormalisedData(dataFilePath, texts):<br>
"""Write normalised data to a file<br>
Input:<br>
dataFilePath : Path to original train/test file that has been processed<br>
texts : List containing the normalised 3 turn conversations, separated by the  tag.<br>
"""<br>
normalisedDataFilePath = dataFilePath.replace(".txt", "_normalised.txt")<br>
with io.open(normalisedDataFilePath, 'w', encoding='utf8') as fout:<br>
with io.open(dataFilePath, encoding='utf8') as fin:<br>
fin.readline()<br>
for lineNum, line in enumerate(fin):<br>
line = line.strip().split('\t')<br>
normalisedLine = texts[lineNum].strip().split('')<br>
fout.write(line[0] + '\t')<br>
# Write the original turn, followed by the normalised version of the same turn<br>
fout.write(line[1] + '\t' + normalisedLine[0] + '\t')<br>
fout.write(line[2] + '\t' + normalisedLine[1] + '\t')<br>
fout.write(line[3] + '\t' + normalisedLine[2] + '\t')<br>
try:<br>
# If label information available (train time)<br>
fout.write(line[4] + '\n')<br>
except:<br>
# If label information not available (test time)<br>
fout.write('\n')</p>
<p dir="auto">def getEmbeddingMatrix(wordIndex):<br>
"""Populate an embedding matrix using a word-index. If the word "happy" has an index 19,<br>
the 19th row in the embedding matrix should contain the embedding vector for the word "happy".<br>
Input:<br>
wordIndex : A dictionary of (word : index) pairs, extracted using a tokeniser<br>
Output:<br>
embeddingMatrix : A matrix where every row has 100 dimensional GloVe embedding<br>
"""<br>
embeddingsIndex = {}<br>
# Load the embedding vectors from ther GloVe file<br>
with io.open(os.path.join(gloveDir, 'glove.6B.100d.txt'), encoding="utf8") as f:<br>
for line in f:<br>
values = line.split()<br>
word = values[0]<br>
embeddingVector = np.asarray(values[1:], dtype='float32')<br>
embeddingsIndex[word] = embeddingVector</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="print('Found %s word vectors.' % len(embeddingsIndex))

# Minimum word index of any word is 1. 
embeddingMatrix = np.zeros((len(wordIndex) + 1, EMBEDDING_DIM))
for word, i in wordIndex.items():
    embeddingVector = embeddingsIndex.get(word)
    if embeddingVector is not None:
        # words not found in embedding index will be all-zeros.
        embeddingMatrix[i] = embeddingVector

return embeddingMatrix"><pre class="notranslate"><code class="notranslate">print('Found %s word vectors.' % len(embeddingsIndex))

# Minimum word index of any word is 1. 
embeddingMatrix = np.zeros((len(wordIndex) + 1, EMBEDDING_DIM))
for word, i in wordIndex.items():
    embeddingVector = embeddingsIndex.get(word)
    if embeddingVector is not None:
        # words not found in embedding index will be all-zeros.
        embeddingMatrix[i] = embeddingVector

return embeddingMatrix
</code></pre></div>
<p dir="auto">def buildModel(embeddingMatrix):<br>
"""Constructs the architecture of the model<br>
Input:<br>
embeddingMatrix : The embedding matrix to be loaded in the embedding layer.<br>
Output:<br>
model : A basic LSTM model<br>
"""<br>
embeddingLayer = Embedding(embeddingMatrix.shape[0],<br>
EMBEDDING_DIM,<br>
weights=[embeddingMatrix],<br>
input_length=MAX_SEQUENCE_LENGTH,<br>
trainable=False)<br>
print(embeddingLayer)<br>
model = Sequential()<br>
model.add(embeddingLayer)<br>
model.add(LSTM(LSTM_DIM, dropout=DROPOUT))<br>
model.add(Dense(NUM_CLASSES, activation='softmax'))</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="rmsprop = optimizers.rmsprop(lr=0.003)
model.compile(loss='categorical_crossentropy',
              optimizer=rmsprop,
              metrics=['acc'])
print(model.summary())
return model"><pre class="notranslate"><code class="notranslate">rmsprop = optimizers.rmsprop(lr=0.003)
model.compile(loss='categorical_crossentropy',
              optimizer=rmsprop,
              metrics=['acc'])
print(model.summary())
return model
</code></pre></div>
<p dir="auto">def main():<br>
'''parser = argparse.ArgumentParser(description="Baseline Script for SemEval")<br>
parser.add_argument('-config', help='Config to read details', required=True)<br>
args = parser.parse_args()</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="with open(args.config) as configfile:
    config = json.load(configfile)
    
global trainDataPath, testDataPath, solutionPath, gloveDir
global NUM_FOLDS, NUM_CLASSES, MAX_NB_WORDS, MAX_SEQUENCE_LENGTH, EMBEDDING_DIM
global BATCH_SIZE, LSTM_DIM, DROPOUT, NUM_EPOCHS, LEARNING_RATE    

trainDataPath = config[&quot;train_data_path&quot;]
testDataPath = config[&quot;test_data_path&quot;]
solutionPath = config[&quot;solution_path&quot;]
gloveDir = config[&quot;glove_dir&quot;]

NUM_FOLDS = config[&quot;num_folds&quot;]
NUM_CLASSES = config[&quot;num_classes&quot;]
MAX_NB_WORDS = config[&quot;max_nb_words&quot;]
MAX_SEQUENCE_LENGTH = config[&quot;max_sequence_length&quot;]
EMBEDDING_DIM = config[&quot;embedding_dim&quot;]
BATCH_SIZE = config[&quot;batch_size&quot;]
LSTM_DIM = config[&quot;lstm_dim&quot;]
DROPOUT = config[&quot;dropout&quot;]
LEARNING_RATE = config[&quot;learning_rate&quot;]
NUM_EPOCHS = config[&quot;num_epochs&quot;]
'''
print(&quot;Processing training data...&quot;)
trainIndices, trainTexts, labels = preprocessData(trainDataPath, mode=&quot;train&quot;)
# Write normalised text to file to check if normalisation works. Disabled now. Uncomment following line to enable   
# writeNormalisedData(trainDataPath, trainTexts)
print(&quot;Processing test data...&quot;)
testIndices, testTexts = preprocessData(testDataPath, mode=&quot;test&quot;)
# writeNormalisedData(testDataPath, testTexts)

print(&quot;Extracting tokens...&quot;)
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(trainTexts)
trainSequences = tokenizer.texts_to_sequences(trainTexts)
testSequences = tokenizer.texts_to_sequences(testTexts)

wordIndex = tokenizer.word_index
print(&quot;Found %s unique tokens.&quot; % len(wordIndex))

print(&quot;Populating embedding matrix...&quot;)
embeddingMatrix = getEmbeddingMatrix(wordIndex)
print(embeddingMatrix)
print('shape of embeddingMatrix',embeddingMatrix.shape)
print('shape of embeddingMatrix.shape[0]: ',embeddingMatrix.shape[0])


data = pad_sequences(trainSequences, maxlen=MAX_SEQUENCE_LENGTH)
labels = to_categorical(np.asarray(labels))
print(&quot;Shape of training data tensor: &quot;, data.shape)
print(&quot;Shape of label tensor: &quot;, labels.shape)
    
# Randomize data
np.random.shuffle(trainIndices)
data = data[trainIndices]
labels = labels[trainIndices]
  
# Perform k-fold cross validation
metrics = {&quot;accuracy&quot; : [],
           &quot;microPrecision&quot; : [],
           &quot;microRecall&quot; : [],
           &quot;microF1&quot; : []}

print(&quot;Starting k-fold cross validation...&quot;)
for k in range(NUM_FOLDS):
    print('-'*40)
    print(&quot;Fold %d/%d&quot; % (k+1, NUM_FOLDS))
    validationSize = int(len(data)/NUM_FOLDS)
    index1 = validationSize * k
    index2 = validationSize * (k+1)
        
    xTrain = np.vstack((data[:index1],data[index2:]))
    yTrain = np.vstack((labels[:index1],labels[index2:]))
    xVal = data[index1:index2]
    yVal = labels[index1:index2]
    print(&quot;Building model...&quot;)
    model = buildModel(embeddingMatrix)
    model.fit(xTrain, yTrain, 
              validation_data=(xVal, yVal),
              epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)

    predictions = model.predict(xVal, batch_size=BATCH_SIZE)
    accuracy, microPrecision, microRecall, microF1 = getMetrics(predictions, yVal)
    metrics[&quot;accuracy&quot;].append(accuracy)
    metrics[&quot;microPrecision&quot;].append(microPrecision)
    metrics[&quot;microRecall&quot;].append(microRecall)
    metrics[&quot;microF1&quot;].append(microF1)
    
print(&quot;\n============= Metrics =================&quot;)
print(&quot;Average Cross-Validation Accuracy : %.4f&quot; % (sum(metrics[&quot;accuracy&quot;])/len(metrics[&quot;accuracy&quot;])))
print(&quot;Average Cross-Validation Micro Precision : %.4f&quot; % (sum(metrics[&quot;microPrecision&quot;])/len(metrics[&quot;microPrecision&quot;])))
print(&quot;Average Cross-Validation Micro Recall : %.4f&quot; % (sum(metrics[&quot;microRecall&quot;])/len(metrics[&quot;microRecall&quot;])))
print(&quot;Average Cross-Validation Micro F1 : %.4f&quot; % (sum(metrics[&quot;microF1&quot;])/len(metrics[&quot;microF1&quot;])))

print(&quot;\n======================================&quot;)

print(&quot;Retraining model on entire data to create solution file&quot;)
model = buildModel(embeddingMatrix)
model.fit(data, labels, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)
model.save('EP%d_LR%de-5_LDim%d_BS%d.h5'%(NUM_EPOCHS, int(LEARNING_RATE*(10**5)), LSTM_DIM, BATCH_SIZE))
model = load_model('EP%d_LR%de-5_LDim%d_BS%d.h5'%(NUM_EPOCHS, int(LEARNING_RATE*(10**5)), LSTM_DIM, BATCH_SIZE))

print(&quot;Creating solution file...&quot;)
testData = pad_sequences(testSequences, maxlen=MAX_SEQUENCE_LENGTH)
predictions = model.predict(testData, batch_size=BATCH_SIZE)
predictions = predictions.argmax(axis=1)

with io.open(solutionPath, &quot;w&quot;, encoding=&quot;utf8&quot;) as fout:
    fout.write('\t'.join([&quot;id&quot;, &quot;turn1&quot;, &quot;turn2&quot;, &quot;turn3&quot;, &quot;label&quot;]) + '\n')        
    with io.open(testDataPath, encoding=&quot;utf8&quot;) as fin:
        fin.readline()
        for lineNum, line in enumerate(fin):
            fout.write('\t'.join(line.strip().split('\t')[:4]) + '\t')
            fout.write(label2emotion[predictions[lineNum]] + '\n')
print(&quot;Completed. Model parameters: &quot;)
print(&quot;Learning rate : %.3f, LSTM Dim : %d, Dropout : %.3f, Batch_size : %d&quot; 
      % (LEARNING_RATE, LSTM_DIM, DROPOUT, BATCH_SIZE))"><pre class="notranslate"><code class="notranslate">with open(args.config) as configfile:
    config = json.load(configfile)
    
global trainDataPath, testDataPath, solutionPath, gloveDir
global NUM_FOLDS, NUM_CLASSES, MAX_NB_WORDS, MAX_SEQUENCE_LENGTH, EMBEDDING_DIM
global BATCH_SIZE, LSTM_DIM, DROPOUT, NUM_EPOCHS, LEARNING_RATE    

trainDataPath = config["train_data_path"]
testDataPath = config["test_data_path"]
solutionPath = config["solution_path"]
gloveDir = config["glove_dir"]

NUM_FOLDS = config["num_folds"]
NUM_CLASSES = config["num_classes"]
MAX_NB_WORDS = config["max_nb_words"]
MAX_SEQUENCE_LENGTH = config["max_sequence_length"]
EMBEDDING_DIM = config["embedding_dim"]
BATCH_SIZE = config["batch_size"]
LSTM_DIM = config["lstm_dim"]
DROPOUT = config["dropout"]
LEARNING_RATE = config["learning_rate"]
NUM_EPOCHS = config["num_epochs"]
'''
print("Processing training data...")
trainIndices, trainTexts, labels = preprocessData(trainDataPath, mode="train")
# Write normalised text to file to check if normalisation works. Disabled now. Uncomment following line to enable   
# writeNormalisedData(trainDataPath, trainTexts)
print("Processing test data...")
testIndices, testTexts = preprocessData(testDataPath, mode="test")
# writeNormalisedData(testDataPath, testTexts)

print("Extracting tokens...")
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(trainTexts)
trainSequences = tokenizer.texts_to_sequences(trainTexts)
testSequences = tokenizer.texts_to_sequences(testTexts)

wordIndex = tokenizer.word_index
print("Found %s unique tokens." % len(wordIndex))

print("Populating embedding matrix...")
embeddingMatrix = getEmbeddingMatrix(wordIndex)
print(embeddingMatrix)
print('shape of embeddingMatrix',embeddingMatrix.shape)
print('shape of embeddingMatrix.shape[0]: ',embeddingMatrix.shape[0])


data = pad_sequences(trainSequences, maxlen=MAX_SEQUENCE_LENGTH)
labels = to_categorical(np.asarray(labels))
print("Shape of training data tensor: ", data.shape)
print("Shape of label tensor: ", labels.shape)
    
# Randomize data
np.random.shuffle(trainIndices)
data = data[trainIndices]
labels = labels[trainIndices]
  
# Perform k-fold cross validation
metrics = {"accuracy" : [],
           "microPrecision" : [],
           "microRecall" : [],
           "microF1" : []}

print("Starting k-fold cross validation...")
for k in range(NUM_FOLDS):
    print('-'*40)
    print("Fold %d/%d" % (k+1, NUM_FOLDS))
    validationSize = int(len(data)/NUM_FOLDS)
    index1 = validationSize * k
    index2 = validationSize * (k+1)
        
    xTrain = np.vstack((data[:index1],data[index2:]))
    yTrain = np.vstack((labels[:index1],labels[index2:]))
    xVal = data[index1:index2]
    yVal = labels[index1:index2]
    print("Building model...")
    model = buildModel(embeddingMatrix)
    model.fit(xTrain, yTrain, 
              validation_data=(xVal, yVal),
              epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)

    predictions = model.predict(xVal, batch_size=BATCH_SIZE)
    accuracy, microPrecision, microRecall, microF1 = getMetrics(predictions, yVal)
    metrics["accuracy"].append(accuracy)
    metrics["microPrecision"].append(microPrecision)
    metrics["microRecall"].append(microRecall)
    metrics["microF1"].append(microF1)
    
print("\n============= Metrics =================")
print("Average Cross-Validation Accuracy : %.4f" % (sum(metrics["accuracy"])/len(metrics["accuracy"])))
print("Average Cross-Validation Micro Precision : %.4f" % (sum(metrics["microPrecision"])/len(metrics["microPrecision"])))
print("Average Cross-Validation Micro Recall : %.4f" % (sum(metrics["microRecall"])/len(metrics["microRecall"])))
print("Average Cross-Validation Micro F1 : %.4f" % (sum(metrics["microF1"])/len(metrics["microF1"])))

print("\n======================================")

print("Retraining model on entire data to create solution file")
model = buildModel(embeddingMatrix)
model.fit(data, labels, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)
model.save('EP%d_LR%de-5_LDim%d_BS%d.h5'%(NUM_EPOCHS, int(LEARNING_RATE*(10**5)), LSTM_DIM, BATCH_SIZE))
model = load_model('EP%d_LR%de-5_LDim%d_BS%d.h5'%(NUM_EPOCHS, int(LEARNING_RATE*(10**5)), LSTM_DIM, BATCH_SIZE))

print("Creating solution file...")
testData = pad_sequences(testSequences, maxlen=MAX_SEQUENCE_LENGTH)
predictions = model.predict(testData, batch_size=BATCH_SIZE)
predictions = predictions.argmax(axis=1)

with io.open(solutionPath, "w", encoding="utf8") as fout:
    fout.write('\t'.join(["id", "turn1", "turn2", "turn3", "label"]) + '\n')        
    with io.open(testDataPath, encoding="utf8") as fin:
        fin.readline()
        for lineNum, line in enumerate(fin):
            fout.write('\t'.join(line.strip().split('\t')[:4]) + '\t')
            fout.write(label2emotion[predictions[lineNum]] + '\n')
print("Completed. Model parameters: ")
print("Learning rate : %.3f, LSTM Dim : %d, Dropout : %.3f, Batch_size : %d" 
      % (LEARNING_RATE, LSTM_DIM, DROPOUT, BATCH_SIZE))
</code></pre></div>
<p dir="auto">if <strong>name</strong> == '<strong>main</strong>':<br>
main()</p>
<p dir="auto">I'm having a weird problem where I am using a doc2vec Embedding layer and even though I'm pumping through unique examples for prediction, a lot of the outputs from the pred_proba method are returning the same prediction.</p>
<p dir="auto">Here is the architecture:</p>
<p dir="auto">print("Loading Doc2Vec...")<br>
model = Doc2Vec.load("doc2vec.model")</p>
<p dir="auto">gensim_dict = Dictionary()<br>
gensim_dict.doc2bow(model.vocab.keys(),<br>
allow_update=True)<br>
w2indx = {v: k+1 for k, v in gensim_dict.items()}<br>
w2vec = {word: model[word] for word in w2indx.keys()}</p>
<h1 dir="auto">index_dict, word_vectors</h1>
<p dir="auto">print('Setting up Arrays for Keras Embedding Layer...')<br>
n_symbols = len(w2indx) + 1  # adding 1 to account for 0th index<br>
embedding_weights = np.zeros((n_symbols, maxlen))<br>
for word, index in w2indx.items():<br>
embedding_weights[index, :] = w2vec[word]</p>
<p dir="auto">model = Sequential()<br>
model.add(Embedding(output_dim=maxlen,<br>
input_dim=n_symbols,<br>
#mask_zero=True,<br>
weights=[embedding_weights],<br>
input_length=maxlen))  # Adding Input Length<br>
model.add(Dropout(0.25))<br>
model.add(Convolution1D(nb_filter=nb_filter,<br>
filter_length=filter_length,<br>
border_mode='valid',<br>
activation='relu',<br>
subsample_length=1))<br>
model.add(MaxPooling1D(pool_length=pool_length))<br>
model.add(LSTM(lstm_output_size))<br>
model.add(Dense(1))<br>
model.add(Activation('sigmoid'))</p>
<p dir="auto">model.compile(loss='binary_crossentropy',<br>
optimizer='adam',<br>
metrics=['accuracy'])</p>
<p dir="auto">print("Loading Weights")<br>
model.load_weights(instrument + '-weights.hdf5')</p>
<p dir="auto">classes = model.predict_proba(X_test, batch_size=1, verbose=0)<br>
print(classes)</p>
<p dir="auto">And the outputs are:</p>
<p dir="auto">Using Theano backend.<br>
X_test:<br>
[[ -2.47382596e-02   2.32602060e-02  -6.53387140e-03 ...,  -5.51625062e-03<br>
-5.74239753e-02  -3.07026859e-02]<br>
[  3.29017431e-01   2.35613093e-01  -7.56662712e-02 ...,   7.71998167e-02<br>
-3.64937872e-01  -2.01133800e+00]<br>
[ -1.00101516e-01  -6.00816719e-02  -3.69741372e-03 ...,   5.17791100e-02<br>
-7.46646821e-02   8.59571397e-02]<br>
...,<br>
[  2.19638228e-01  -8.07978734e-02   4.71501723e-02 ...,   4.79124673e-02<br>
1.00293700e-02   1.53316772e-02]<br>
[ -1.25041693e-01  -6.55420348e-02   4.24071133e-01 ...,   2.15420816e-02<br>
-5.21663465e-02  -7.86688030e-02]<br>
[ -4.78273164e-03   3.24305370e-02   2.72341162e-01 ...,  -8.28378797e-02<br>
1.43599689e-01  -1.07673614e-03]]</p>
<p dir="auto">Loading Doc2Vec...<br>
Setting up Arrays for Keras Embedding Layer...<br>
Loading Weights</p>
<p dir="auto">Prediction:</p>
<p dir="auto">[[ 0.50939184]<br>
[ 0.34107986]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.48695773]<br>
[ 0.50939184]<br>
[ 0.62477738]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.58683521]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]<br>
[ 0.50939184]</p>
<p dir="auto">You can see 0.50939184 appears over and over again.</p>