text_plain_output_1.png
from keras.preprocessing.text import Tokenizer

import os

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import string

import numpy as np
import pandas as pd
import os

currr_dir = '../input/'
all_headlines = []
x = 0
for filename in os.listdir(currr_dir):
    if 'Articles' in filename:
        article_df = pd.read_csv(currr_dir + filename)
        all_headlines.extend(list(article_df.headline.values))
        x = 1
        break
all_headlines = [h for h in all_headlines if h != 'Unknown']

def clean_text(txt):
    txt = ''.join((w for w in txt if w not in string.punctuation)).lower()
    txt = txt.encode('utf8').decode('ascii', 'ignore')
    return txt

corpus = [clean_text(x) for x in all_headlines]

tokenizer = Tokenizer()
def get_sequence_of_tokens(corpus):
    q = 0
    tokenizer.fit_on_texts(corpus)
    total_words = len(tokenizer.word_index) + 1
    input_sequences = []
    for line in corpus:
        token_list = tokenizer.texts_to_sequences([line])[0]
        for i in range(1, len(token_list)):
            n_gram_sequence = token_list[:i + 1]
            input_sequences.append(n_gram_sequence)
            q = 1
    return (input_sequences, total_words)

input_sequence, total_words = get_sequence_of_tokens(corpus)
print(total_words)