text_plain_output_1.png
from nltk import FreqDist, bigrams, trigrams

from nltk.corpus import stopwords

import pandas as pd

grouped = data.groupby(['name', 'address']).agg({'listed_in(type)': list})
newdata = pd.merge(grouped, data, on=['name', 'address'])
newdata['listed_in(type)_x'] = newdata['listed_in(type)_x'].astype(str)
newdata.drop_duplicates(subset=['name', 'address', 'listed_in(type)_x'], inplace=True)
newdata = newdata.reset_index(drop=True)

newdata['rating'] = newdata['rate'].str[:3]
newdata = newdata[newdata.rating != 'NEW']
newdata = newdata.dropna(subset=['rating'])
newdata['rating'] = pd.to_numeric(newdata['rating'])
newdata['rating'] = pd.cut(newdata['rating'], bins=[0, 3.0, 3.5, 4.0, 5.0], labels=['0', '1', '2', '3'])

reviews_data = newdata[['reviews_list', 'rating']]
reviews_data['reviews_list'][0]

reviews_data['reviews_list'] = reviews_data['reviews_list'].apply(lambda x: x.lower())
tokenizer = regextoken('[a-zA-Z]+')
review_tokens = reviews_data['reviews_list'].apply(tokenizer.tokenize)

stop = stopwords.words('english')

stop.extend(['rated', 'n', 'nan', 'x'])
review_tokens = review_tokens.apply(lambda x: [token for token in x if token not in stop])
all_reviews = review_tokens.astype(str).str.cat()
cleaned_reviews = tokenizer.tokenize(all_reviews)
fd = FreqDist()
for word in cleaned_reviews:
    fd[word] += 1
fd.most_common(5)

trigrams = trigrams(cleaned_reviews)
fd_trigrams = FreqDist()
for trigram in trigrams:
    fd_trigrams[trigram] += 1
fd_trigrams.most_common(5)