text_html_output_1.png
from nltk.corpus import stopwords

import pandas as pd

import string

import numpy as np
import pandas as pd
import re
import nltk
import spacy
import string
pd.options.mode.chained_assignment = None
full_df = pd.read_csv('../input/twcs/twcs.csv', nrows=5000)
df = full_df[['text']]
df['text'] = df['text'].astype(str)

df['text_lower'] = df['text'].str.lower()

df.drop(['text_lower'], axis=1, inplace=True)
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
    """custom function to remove the punctuation"""
    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df['text_wo_punct'] = df['text'].apply(lambda text: remove_punctuation(text))

from nltk.corpus import stopwords
', '.join(stopwords.words('english'))

STOPWORDS = set(stopwords.words('english'))
def remove_stopwords(text):
    """custom function to remove the stopwords"""
    return ' '.join([word for word in str(text).split() if word not in STOPWORDS])
df['text_wo_stop'] = df['text_wo_punct'].apply(lambda text: remove_stopwords(text))
df.head()