text_html_output_1.png
from collections import Counter

from nltk.corpus import stopwords

import pandas as pd

import string

import numpy as np
import pandas as pd
import re
import nltk
import spacy
import string
pd.options.mode.chained_assignment = None
full_df = pd.read_csv('../input/twcs/twcs.csv', nrows=5000)
df = full_df[['text']]
df['text'] = df['text'].astype(str)

df['text_lower'] = df['text'].str.lower()

df.drop(['text_lower'], axis=1, inplace=True)
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
    """custom function to remove the punctuation"""
    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df['text_wo_punct'] = df['text'].apply(lambda text: remove_punctuation(text))

from nltk.corpus import stopwords
', '.join(stopwords.words('english'))

STOPWORDS = set(stopwords.words('english'))
def remove_stopwords(text):
    """custom function to remove the stopwords"""
    return ' '.join([word for word in str(text).split() if word not in STOPWORDS])
df['text_wo_stop'] = df['text_wo_punct'].apply(lambda text: remove_stopwords(text))

from collections import Counter
cnt = Counter()
for text in df['text_wo_stop'].values:
    for word in text.split():
        cnt[word] += 1
cnt.most_common(10)