text_plain_output_1.png
from keras.layers import LSTM

from keras.layers.core import Activation, Dropout, Dense

from keras.layers.embeddings import Embedding

from keras.models import Sequential

from keras.preprocessing.sequence import pad_sequences

from keras.preprocessing.text import Tokenizer

from nltk.corpus import stopwords

from tensorflow.keras.optimizers import Adam

import emoji

import numpy as np

import numpy as np # linear algebra

import re # regular expression operations

import wordninja

stop_words = stopwords.words('english')
abbreviations = {'a.m.': 'before midday', 'acct': 'account', 'afaik': 'as far as i know', 'afk': 'away from keyboard', 'app': 'application', 'approx': 'approximately', 'apps': 'applications', 'asap': 'as soon as possible', 'asl': 'age, sex, location', 'ave.': 'avenue', 'b.c': 'before christ', 'b2b': 'business to business', 'b4': 'before', 'bae': 'before anyone else', 'bak': 'back at keyboard', 'bbl': 'be back later', 'bbs': 'be back soon', 'be4': 'before', 'blvd': 'boulevard', 'bout': 'about', 'brb': 'be right back', 'btw': 'by the way', 'c/o': 'care of', 'cf': 'compare', 'cia': 'central intelligence agency', 'cu': 'see you', 'cya': 'see you', 'dae': 'does anyone else', 'diy': 'do it yourself', 'dm': 'direct message', 'eet': 'eastern european time', 'eg': 'example', 'encl.': 'enclosed', 'etc': 'and so on', 'fb': 'facebook', 'fc': 'fingers crossed', 'fig': 'figure', 'fwiw': 'for what it is worth', 'fyi': 'for your information', 'ft.': 'feet', 'ft': 'featuring', 'ftl': 'for the loss', 'ftw': 'for the win', 'gal': 'get a life', 'gcse': 'general certificate of secondary education', 'gg': 'good game', 'gl': 'good luck', 'gmt': 'greenwich mean time', 'gn': 'good night', 'goat': 'greatest of all time', 'gps': 'global positioning system', 'gr8': 'great', 'gratz': 'congratulations', 'gyal': 'girl', 'irl': 'in real life', 'hp': 'horsepower', 'hr': 'hour', 'ht': 'height', 'ic': 'i see', 'idc': 'i do not care', 'idgaf': 'i do not give a fuck', 'idk': 'i do not know', 'ie': 'that is', 'iirc': 'if i remember correctly', 'ilu': 'i love you', 'ily': 'i love you', 'imho': 'in my humble opinion', 'imo': 'in my opinion', 'iow': 'in other words', 'jk': 'just kidding', 'l8r': 'later', 'lb': 'pound', 'lbs': 'pounds', 'lmao': 'laugh my ass off', 'lmfao': 'laugh my fucking ass off', 'lol': 'laughing out loud', 'ltd': 'limited', 'm8': 'mate', 'mf': 'motherfucker', 'mofo': 'motherfucker', 'mph': 'miles per hour', 'mr': 'mister', 'ms': 'miss', 'mte': 'my thoughts exactly', 'nbc': 'national broadcasting company', 'nfs': 'not for sale', 'ngl': 'not going to lie', 'nhs': 'national health service', 'nsfw': 'not safe for work', 'rt': 'retweet', 'nth': 'nice to have', 'nvr': 'never', 'nyc': 'new york city', 'oc': 'original content', 'og': 'original', 'oic': 'oh i see', 'omg': 'oh my god', 'omw': 'on my way', 'p.m': 'after midday', 'poc': 'people of color', 'pov': 'point of view', 'pp': 'pages', 'ppl': 'people', 'ps': 'postscript', 'pt': 'point', 'pto': 'please turn over', 'ratchet': 'rude', 'roflol': 'rolling on the floor laughing out loud', 'sk8': 'skate', 'smh': 'shake my head', 'sq': 'square', 'tbh': 'to be honest', 'tbs': 'tablespooful', 'thks': 'thank you', 'tho': 'though', 'thx': 'thank you', 'tia': 'thanks in advance', 'til': 'today i learned', 'tl;dr': 'too long i did not read', 'u2': 'you too', 'w/': 'with', 'tmb': 'tweet me back', 'ttyl': 'talk to you later', 'u': 'you', 'w/o': 'without', 'w8': 'wait', 'wassup': 'what is up', 'wb': 'welcome back', 'wtf': 'what the fuck', 'wtg': 'way to go', 'some1': 'someone', 'yrs': 'years', 'hrs': 'hours', '2morow': 'tomorrow', '2moro': 'tomorrow', '2day': 'today', '4got': 'forget', 'bday': 'birthday', 'b-day': 'birthday'}
def preprocess(text, wordLemm):
    processedText = []
    for tweet in text:
        tweet = tweet.lower()
        tweet = re.sub('((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)', 'url', tweet)
        tweet = emoji.demojize(tweet)
        tweet = re.sub('@[^\\s]+', 'user', tweet)
        splitted = wordninja.split(tweet)
        tweet = ' '.join((word for word in splitted))
        for s in abbreviations.keys():
            tweet = tweet.replace(' ' + s, ' ' + abbreviations[s])
        tweet = re.sub('[^a-z\\s]', '', tweet)
        tweet = re.sub('(.)\\1\\1+', '\\1', tweet)
        tweetwords = ''
        for word in tweet.split():
            if len(word) > 1:
                word = wordLemm.lemmatize(word)
                tweetwords += word + ' '
        processedText.append(tweetwords)
    return processedText

tokenizer = Tokenizer()
tokenizer.fit_on_texts(X_train)
word_index = tokenizer.word_index
X_train_tok = tokenizer.texts_to_sequences(X_train)
X_test_tok = tokenizer.texts_to_sequences(X_test)

vocab_size = len(tokenizer.word_index) + 1
X_train_tok = pad_sequences(X_train_tok, padding='post', maxlen=30)
X_test_tok = pad_sequences(X_test_tok, padding='post', maxlen=30)

embeddings_dict = {}
glove_file = open('glove.6B.300d.txt', encoding='utf8')
for line in glove_file:
    records = line.split()
    word = records[0]
    vector_dimensions = np.asarray(records[1:], dtype='float32')
    embeddings_dict[word] = vector_dimensions
glove_file.close()

embedding_matrix = np.zeros((vocab_size, 300))
for word, i in word_index.items():
    embedding_vector = embeddings_dict.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector

embedding_layer = Embedding(vocab_size, 300, weights=[embedding_matrix], input_length=30, trainable=False)

model2 = Sequential()
model2.add(embedding_layer)
model2.add(Dropout(0.5))
model2.add(LSTM(128))
model2.add(Dropout(0.5))
model2.add(Dense(512, activation='relu'))
model2.add(Dropout(0.5))
model2.add(Dense(1, activation='sigmoid'))

model2.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=['accuracy'])

history = model2.fit(X_train_tok, y_train, batch_size=128, epochs=10, verbose=1, validation_split=0.2)