text_html_output_2.png
from tensorflow.keras.preprocessing.text import Tokenizer

import os , string , sys , numpy, spacy , nltk, re, random, timeit

import pandas as pd

working_dir = '../input/nyt-comments/'
headlines = []
for filename in os.listdir(working_dir):
    if 'Articles' in filename:
        article_df = pd.read_csv(working_dir + filename)
        headlines.extend(list(article_df.headline.values))
all_headlines = [x for x in headlines if x != 'Unknown']
all_headlines = headlines

def clean_text(txt):
    txt = ''.join((v for v in txt if v not in string.punctuation)).lower()
    txt = txt.encode('utf8').decode('ascii', 'ignore')
    return txt

corpus = [clean_text(x) for x in headlines]

tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1
print(f'total words: {total_words}')