text_html_output_1.png
import gensim

import pandas as pd

import pandas as pd

import re

data = pd.read_csv('../input/amazon-fine-food-reviews/Reviews.csv', index_col='Id')

data = data.drop({'ProductId', 'UserId', 'ProfileName', 'HelpfulnessNumerator', 'HelpfulnessDenominator', 'Time', 'Summary'}, axis=1)

data.Score = ['positive' if i >= 4 else 'negative' for i in data.Score]

import re
data.Text = [i.lower() for i in data.Text]
data.Text = [re.sub('[^\\w\\s]', '', i) for i in data.Text]

def preprocess_text(text):
    tokens = gensim.utils.simple_preprocess(text)
    return tokens
data['tokens'] = data['Text'].apply(preprocess_text)
data.head(25)