text_plain_output_1.png
from keras.layers import Embedding, Dense, Dropout, LSTM

from keras.models import Sequential

from keras.preprocessing.sequence import pad_sequences

from keras.preprocessing.text import Tokenizer

import keras.utils as ku

import numpy as np # linear algebra

import os

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import string

import numpy as np
import pandas as pd
import os

currr_dir = '../input/'
all_headlines = []
x = 0
for filename in os.listdir(currr_dir):
    if 'Articles' in filename:
        article_df = pd.read_csv(currr_dir + filename)
        all_headlines.extend(list(article_df.headline.values))
        x = 1
        break
all_headlines = [h for h in all_headlines if h != 'Unknown']

def clean_text(txt):
    txt = ''.join((w for w in txt if w not in string.punctuation)).lower()
    txt = txt.encode('utf8').decode('ascii', 'ignore')
    return txt

corpus = [clean_text(x) for x in all_headlines]

tokenizer = Tokenizer()
def get_sequence_of_tokens(corpus):
    q = 0
    tokenizer.fit_on_texts(corpus)
    total_words = len(tokenizer.word_index) + 1
    input_sequences = []
    for line in corpus:
        token_list = tokenizer.texts_to_sequences([line])[0]
        for i in range(1, len(token_list)):
            n_gram_sequence = token_list[:i + 1]
            input_sequences.append(n_gram_sequence)
            q = 1
    return (input_sequences, total_words)

def generate_padded_sequences(input_sequence):
    max_sequence_len = max([len(x) for x in input_sequence])
    input_sequences = np.array(pad_sequences(input_sequence, maxlen=max_sequence_len, padding='pre'))
    predictors, labels = (input_sequences[:, :-1], input_sequences[:, -1])
    labels = ku.to_categorical(labels, num_classes=total_words)
    return (predictors, labels, max_sequence_len)

def create_model(max_seq_len, total_words):
    input_len = max_seq_len - 1
    model = Sequential()
    model.add(Embedding(total_words, 10, input_length=input_len))
    model.add(LSTM(100))
    model.add(Dropout(0.1))
    model.add(Dense(total_words, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    return model

model = create_model(max_seq_len, total_words)
model.summary()

model.fit(predictors, labels, epochs=100, verbose=5)