text_html_output_1.png
from nltk.corpus import stopwords

from nltk.stem.porter import PorterStemmer

from nltk.tokenize import word_tokenize

import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import string

data_df = pd.read_csv('../input/imdb-dataset-of-50k-movie-reviews/IMDB Dataset.csv')

data_df.shape

for i in range(10):
    idx = np.random.randint(1, 50001)

def remove_break(text):
    return re.sub('<br />', '', text)
def remove_punct(text):
    nopunct = ''
    for c in text:
        if c not in string.punctuation:
            nopunct = nopunct + c
    return nopunct
def remove_numbers(text):
    return re.sub('[0-9]', '', text)
def remove_links(text):
    return re.sub('http\\S+', '', text)
def remove_stop_words(word_list):
    stopwords_list = set(stopwords.words('english'))
    word_list = [word for word in word_list if word not in stopwords_list]
    return ' '.join(word_list)
def get_root(word_list):
    ps = PorterStemmer()
    return [ps.stem(word) for word in word_list]
def clean_text(text):
    text = remove_break(text)
    text = remove_links(text)
    text = remove_numbers(text)
    text = remove_punct(text)
    word_list = word_tokenize(text)
    word_list = get_root(word_list)
    return ' '.join(word_list)

data_df['clean_review'] = data_df['review'].progress_apply(clean_text)