text_plain_output_1.png
from keras.callbacks import EarlyStopping

from keras.layers import CuDNNLSTM, Activation, Dense, Dropout, Input, Embedding, concatenate, Bidirectional

from keras.models import Sequential, Model

from keras.optimizers import adam

from keras.preprocessing import sequence

from keras.preprocessing.text import Tokenizer

from keras.utils import np_utils

import numpy as np

import pandas as pd

import re

train_data = pd.read_csv('../input/train.csv')
train_data.sample(5)

train_data['target'].value_counts()[0] / train_data.shape[0] * 100

train_data.isna().sum()

train_data.drop(['id'], axis=1, inplace=True)

"""Adding additional informative columns, as most toxic tweets contain 
   exclamations, capitalized words that can serve as important markers"""
regex = re.compile('[@_!#$%^&*()<>?/\\|}{~:]')
train_data['capitals'] = train_data['comment_text'].apply(lambda x: sum((1 for c in x if c.isupper())))
train_data['exclamation_points'] = train_data['comment_text'].apply(lambda x: len(regex.findall(x)))
train_data['total_length'] = train_data['comment_text'].apply(len)

features_added = ('capitals', 'exclamation_points', 'total_length')
features_existing = ('target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat', 'funny', 'wow', 'sad', 'likes', 'disagree', 'sexual_explicit', 'identity_annotator_count', 'toxicity_annotator_count')
rows = [{c: train_data[f].corr(train_data[c]) for c in features_existing} for f in features_added]
train_correlations = pd.DataFrame(rows, index=features_added)

X = train_data['comment_text']
y = train_data['target']

max_words = 10000
tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(X)

y_train = np_utils.to_categorical(y_train, num_classes=2)
y_test = np_utils.to_categorical(y_test, num_classes=2)

aux_columns = ['capitals', 'exclamation_points', 'total_length']

X = train_data[aux_columns + ['comment_text']]
y = train_data['target']

train_meta_features = np.asarray(x_train[aux_columns])
test_meta_features = np.asarray(x_test[aux_columns])

x_train = x_train['comment_text']
x_test = x_test['comment_text']

y_train = np_utils.to_categorical(y_train, num_classes=2)
y_test = np_utils.to_categorical(y_test, num_classes=2)

max_words = 10000
tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(x_train)
sequences = tok.texts_to_sequences(x_train)
sequences_matrix = sequence.pad_sequences(sequences)

max_len = sequences_matrix.shape[1]
def Meta_RNN():
    nlp_input = Input(shape=(max_len,), name='nlp_input')
    meta_input = Input(shape=(3,), name='meta_input')
    embedding_layer = Embedding(max_words, 64, input_length=max_len)(nlp_input)
    nlp_out = Bidirectional(CuDNNLSTM(64))(embedding_layer)
    combined_input = concatenate([nlp_out, meta_input])
    layer = Dense(256, name='fc1')(combined_input)
    layer = Activation('relu')(layer)
    layer = Dropout(0.5)(layer)
    layer = Dense(2, name='out_layer')(layer)
    layer = Activation('softmax')(layer)
    model = Model(inputs=[nlp_input, meta_input], outputs=layer)
    return model
model = Meta_RNN()
model.summary()
model.compile(loss='categorical_crossentropy', optimizer=adam(lr=0.001, amsgrad=True), metrics=['accuracy'])

model.fit([sequences_matrix, train_meta_features], y_train, batch_size=128, epochs=5, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', min_delta=0.0001)])

test_sequences = tok.texts_to_sequences(x_test)
test_sequences_matrix = sequence.pad_sequences(test_sequences, maxlen=max_len)

score = model.evaluate([test_sequences_matrix, test_meta_features], y_test, verbose=True)

X = train_data['comment_text']
y = train_data['target']

data_sequences = tok.texts_to_sequences(X)
data_matrix = sequence.pad_sequences(data_sequences)
y = np_utils.to_categorical(y, num_classes=2)

X_meta_features = np.asarray(train_data[aux_columns])

max_len = data_matrix.shape[1]
model = Meta_RNN()
model.compile(loss='binary_crossentropy', optimizer=adam(lr=0.001, amsgrad=True), metrics=['accuracy'])
model.fit([data_matrix, X_meta_features], y, batch_size=512, epochs=5, validation_split=0.0, verbose=True)

test_data = pd.read_csv('../input/test.csv')

meta_features = np.asarray(test_data[aux_columns])

data = test_data['comment_text']
sequences = tok.texts_to_sequences(data)
test_sequences_matrix = sequence.pad_sequences(sequences, maxlen=max_len)

print(test_sequences_matrix.shape)
print(meta_features.shape)