text_plain_output_1.png
from tensorflow import keras

from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, Dropout

from tensorflow.keras.models import Sequential

from tensorflow.keras.preprocessing.sequence import pad_sequences

from tensorflow.keras.preprocessing.text import Tokenizer

import matplotlib.pyplot as plt

import numpy as np

import os , string , sys , numpy, spacy , nltk, re, random, timeit

import pandas as pd

import tensorflow as tf

working_dir = '../input/nyt-comments/'
headlines = []
for filename in os.listdir(working_dir):
    if 'Articles' in filename:
        article_df = pd.read_csv(working_dir + filename)
        headlines.extend(list(article_df.headline.values))
all_headlines = [x for x in headlines if x != 'Unknown']
all_headlines = headlines

full_txt = ' '.join(map(str, headlines))

nlp = spacy.load('en_core_web_sm')
stopword = nltk.corpus.stopwords.words('english')
def text_cleaning(text):
    text = re.sub('[^\\w\\s]', '', str(text))
    text = re.split('\\W+', text)
    text = [word for word in text if word not in stopword]
    text = ' '.join(text)
    return text
def frequent_of_words(string):
    clean_string = text_cleaning(string)
    split_string = pd.DataFrame(clean_string.split(), columns=['Words'])
    split_string = split_string.value_counts()[:1000].reset_index(drop=False)[:1000]
    split_string.columns = ['Words', 'Count']
    return split_string
frequent_words = frequent_of_words(full_txt)
frequent_words[:15].style.background_gradient(cmap='Blues')

name_list = ['Trump', 'Obama']
scripts = []
split_string = full_txt.split()
for name in name_list:
    scripts.append((name, split_string.count(name)))
colors = ['#2F86A6', '#F2F013']
sections = [scripts[0][1], scripts[1][1]]
plt.axis('equal')

def clean_text(txt):
    txt = ''.join((v for v in txt if v not in string.punctuation)).lower()
    txt = txt.encode('utf8').decode('ascii', 'ignore')
    return txt

corpus = [clean_text(x) for x in headlines]

tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1

input_sequences = []
for line in corpus:
    token_list = tokenizer.texts_to_sequences([line])[0]
    for i in range(1, len(token_list)):
        n_gram_sequence = token_list[:i + 1]
        input_sequences.append(n_gram_sequence)
max_sequence_len = max([len(x) for x in input_sequences])
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))
xs, labels = (input_sequences[:, :-1], input_sequences[:, -1])
ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)

embedding_dim = 100
lstm_units = 150
learning_rate = 0.01
model = Sequential([Embedding(total_words, embedding_dim, input_length=max_sequence_len - 1), Bidirectional(LSTM(lstm_units)), Dense(total_words, activation='softmax')])
model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate), metrics=['accuracy'])

history = model.fit(xs, ys, epochs=100)

def plot_loss_curves(history):
    """
      returns seperate loss curves for training and validation metrics
    """
    train_loss = history.history['loss']
    train_accuracy = history.history['accuracy']
    epochs = range(1, len(history.history['loss']) + 1)
    plt.figure(figsize=(8, 3))
    plt.subplot(1, 2, 2)
    plt.plot(epochs, train_accuracy, label='training_acc')
    plt.title('Accuracy curves', size=5)
    plt.xlabel('epochs', size=5)
    plt.ylabel('Accuracy', size=5)
    plt.tight_layout()
    plt.legend(fontsize=10)
    plt.subplot(1, 2, 1)
    plt.plot(epochs, train_loss, label='training_loss')
    plt.title('Loss curves', size=5)
    plt.xlabel('epochs', size=5)
    plt.ylabel('loss', size=5)
    plt.legend(fontsize=10)
    plt.title('Model Performance Curves')
plot_loss_curves(history)