application_vnd.jupyter.stderr_output_1.png
import os , string , sys , numpy, spacy , nltk, re, random, timeit

import pandas as pd

working_dir = '../input/nyt-comments/'
headlines = []
for filename in os.listdir(working_dir):
    if 'Articles' in filename:
        article_df = pd.read_csv(working_dir + filename)
        headlines.extend(list(article_df.headline.values))
all_headlines = [x for x in headlines if x != 'Unknown']
all_headlines = headlines

full_txt = ' '.join(map(str, headlines))

nlp = spacy.load('en_core_web_sm')
stopword = nltk.corpus.stopwords.words('english')
def text_cleaning(text):
    text = re.sub('[^\\w\\s]', '', str(text))
    text = re.split('\\W+', text)
    text = [word for word in text if word not in stopword]
    text = ' '.join(text)
    return text
def frequent_of_words(string):
    clean_string = text_cleaning(string)
    split_string = pd.DataFrame(clean_string.split(), columns=['Words'])
    split_string = split_string.value_counts()[:1000].reset_index(drop=False)[:1000]
    split_string.columns = ['Words', 'Count']
    return split_string
frequent_words = frequent_of_words(full_txt)
frequent_words[:15].style.background_gradient(cmap='Blues')