text_html_output_1.png
from datetime import datetime

from keras.layers import Input, Embedding, Dense, Bidirectional, CuDNNGRU, GlobalMaxPooling1D

from keras.models import Model

from keras.optimizers import Adamax

from keras.preprocessing.sequence import pad_sequences

from keras.preprocessing.text import Tokenizer

from sklearn.metrics import accuracy_score

from sklearn.metrics import confusion_matrix

from sklearn.model_selection import StratifiedKFold

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

import numpy as np

import os

import pandas as pd

import plotly

import plotly.graph_objs as go

import re

import seaborn as sns

import seaborn as sns

import warnings

import os
import numpy as np
import pandas as pd
import re
from datetime import datetime
import seaborn as sns
import matplotlib.pyplot as plt
import plotly
import plotly.plotly as py
import plotly.graph_objs as go
import colorlover as cl
plotly.offline.init_notebook_mode()
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from sklearn.decomposition import PCA
from keras.models import Model
from keras.optimizers import Adamax
from sklearn.metrics import log_loss
from keras.utils import to_categorical
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Input, Embedding, Dense, Bidirectional, CuDNNGRU, GlobalMaxPooling1D
import seaborn as sns
import matplotlib.pyplot as plt

def clean(text):
    text = re.sub('#\\S*', ' ', text)
    text = re.sub('@\\S*', ' ', text)
    text = re.sub('http\\S*', ' ', text)
    for ch in ['\\', '`', '*', '_', '{', '}', '[', ']', '(', ')', '>', '+', '-', '.', '!', "'", '\\”', '"', '\\“', '\\’', '?', ':', '-', ',', '//t', '&amp;', '/', "'", "'", '…', '-', '’', '\\—', '—', '–', '“', '”']:
        if ch in text:
            text = text.replace(ch, ' ')
    return text
def clean_tweet(tweet):
    return ' '.join(clean(tweet.lower()).split())
def print_table(header_values, content, colors):
    data = go.Table(header=dict(values=header_values, line=dict(color='rgb(70,130,180)'), fill=dict(color='rgb(70,130,180)'), align='center', font=dict(color='black', size=9)), cells=dict(values=content, fill=colors, align='center', font=dict(color='black', size=9), height=40))
def load_glove(word_index):
    EMBEDDING_FILE = '../input/glove840b300dtxt/glove.840B.300d.txt'
    def get_coefs(word, *arr):
        return (word, np.asarray(arr, dtype='float32'))
    embeddings_index = dict((get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE)))
    all_embs = np.stack(embeddings_index.values())
    emb_mean, emb_std = (all_embs.mean(), all_embs.std())
    embed_size = all_embs.shape[1]
    nb_words = min(18000, len(word_index))
    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))
    for word, i in word_index.items():
        if i >= 18000:
            continue
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    return embedding_matrix
def load_data(X, y, train_index, test_index):
    Xtr = X.iloc[train_index]
    Xte = X.iloc[test_index]
    tokenizer = Tokenizer(num_words=18000)
    tokenizer.fit_on_texts(list(Xtr))
    Xtr = tokenizer.texts_to_sequences(Xtr)
    Xte = tokenizer.texts_to_sequences(Xte)
    Xtr = pad_sequences(Xtr, maxlen=50)
    Xte = pad_sequences(Xte, maxlen=50)
    ytr = y.iloc[train_index]
    yte = y.iloc[test_index]
    word_index = tokenizer.word_index
    return (Xtr, ytr, Xte, yte, word_index)

twitter_files = os.listdir('../input/2020-united-states-presidential-election/twitter')
twitter_users_files = os.listdir('../input/2020-united-states-presidential-election/twitter_users')
pic_files = os.listdir('../input/2020-united-states-presidential-election/pics')
metadata = pd.read_csv('../input/2020-united-states-presidential-election/candidates_info.csv')
metadata['filename'] = metadata['handle'].apply(lambda x: x[1:])
metadata['age'] = ((datetime.today() - pd.to_datetime(metadata['born'])).dt.days / 365).astype(int)
metadata = metadata.sort_values('filename')
metadata.reset_index(inplace=True)
dataset = pd.DataFrame()
for index, row in metadata.iterrows():
    df = pd.read_csv('../input/2020-united-states-presidential-election/twitter/%s.csv' % row['filename'])
    dataset = pd.concat([dataset, df], ignore_index=True)
dataset['clean tweet'] = dataset['Text'].apply(clean_tweet)
dataset['number of characters'] = dataset['clean tweet'].str.len()
metadata['number of all tweets'] = dataset.groupby(['Screen Name']).count()['Tweet Id'].values
metadata['tweets in english'] = dataset[dataset['Language'] == 'English'].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['tweets not in english'] = dataset[dataset['Language'] != 'English'].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['tweets only'] = dataset[dataset['Tweet Type'] == 'Tweet'].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['retweets only'] = dataset[dataset['Tweet Type'] == 'Retweet'].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['replies only'] = dataset[dataset['Tweet Type'] == 'Reply'].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['after 2019'] = dataset[dataset['Created At'].astype('datetime64').dt.year > 2018].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['before 2019'] = metadata['number of all tweets'] - metadata['after 2019']
metadata['more than 40 characters'] = dataset[dataset['number of characters'] > 39].groupby(['Screen Name']).count()['Tweet Id'].values
metadata['less than 40 characters'] = dataset[dataset['number of characters'] < 40].groupby(['Screen Name']).count()['Tweet Id'].values
dataset = dataset[dataset['Created At'].astype('datetime64').dt.year > 2018]
dataset = dataset[dataset['number of characters'] > 39]
dataset = dataset[dataset['Tweet Type'] == 'Tweet']
dataset = dataset[dataset['Language'] == 'English']
X = dataset['clean tweet']
y = dataset['Name']
metadata['useful tweets'] = dataset.groupby(['Screen Name']).count()['Tweet Id'].values

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
kfold = StratifiedKFold(n_splits=10, random_state=0, shuffle=True)
scores = []
test_list = []
predict_list = []
test_df = pd.DataFrame()
pred_df = pd.DataFrame()
for train_index, test_index in kfold.split(X, y):
    yoh = pd.get_dummies(y)
    train_X, train_y, test_X, test_y, word_index = load_data(X, yoh, train_index, test_index)
    embedding = load_glove(word_index)
    inp = Input(shape=(50,))
    x = Embedding(18000, 300, weights=[embedding])(inp)
    x = Bidirectional(CuDNNGRU(256, return_sequences=True))(x)
    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
    x = GlobalMaxPooling1D()(x)
    x = Dense(train_y.shape[1], activation='softmax')(x)
    model = Model(inputs=inp, outputs=x)
    model.compile(loss='categorical_crossentropy', optimizer=Adamax(lr=0.002), metrics=['accuracy'])
    model.fit(train_X, train_y, batch_size=1024, epochs=15, validation_data=(test_X, test_y), verbose=0)
    pred_y = model.predict(test_X)
    test_df = pd.concat([test_df, test_y], axis=0, ignore_index=True)
    pred_df = pd.concat([pred_df, pd.DataFrame(pred_y)], axis=0, ignore_index=True)
pred_df.columns = test_df.columns.values
test_label = test_df.idxmax(axis=1)
pred_label = pred_df.idxmax(axis=1)

cm = confusion_matrix(test_label, pred_label, test_df.columns.values)
cm = (cm.astype('float') * 100 / cm.sum(axis=1)[:, np.newaxis] + 0.5).astype('int') / 100
f, ax = plt.subplots(figsize=(14, 12))
sns.set(font_scale=1.4)
sns.heatmap(cm, annot=True, annot_kws={'size': 8}, xticklabels=test_df.columns.values, yticklabels=test_df.columns.values)