text_plain_output_1.png
from keras.preprocessing.text import Tokenizer

from nltk import FreqDist, bigrams, trigrams

from nltk import WordNetLemmatizer

from nltk.corpus import stopwords

from sklearn.model_selection import train_test_split

from sklearn.preprocessing import LabelEncoder

import pandas as pd

grouped = data.groupby(['name', 'address']).agg({'listed_in(type)': list})
newdata = pd.merge(grouped, data, on=['name', 'address'])
newdata['listed_in(type)_x'] = newdata['listed_in(type)_x'].astype(str)
newdata.drop_duplicates(subset=['name', 'address', 'listed_in(type)_x'], inplace=True)
newdata = newdata.reset_index(drop=True)

newdata['rating'] = newdata['rate'].str[:3]
newdata = newdata[newdata.rating != 'NEW']
newdata = newdata.dropna(subset=['rating'])
newdata['rating'] = pd.to_numeric(newdata['rating'])
newdata['rating'] = pd.cut(newdata['rating'], bins=[0, 3.0, 3.5, 4.0, 5.0], labels=['0', '1', '2', '3'])

reviews_data = newdata[['reviews_list', 'rating']]
reviews_data['reviews_list'][0]

reviews_data['reviews_list'] = reviews_data['reviews_list'].apply(lambda x: x.lower())
tokenizer = regextoken('[a-zA-Z]+')
review_tokens = reviews_data['reviews_list'].apply(tokenizer.tokenize)

stop = stopwords.words('english')

stop.extend(['rated', 'n', 'nan', 'x'])
review_tokens = review_tokens.apply(lambda x: [token for token in x if token not in stop])
all_reviews = review_tokens.astype(str).str.cat()
cleaned_reviews = tokenizer.tokenize(all_reviews)
fd = FreqDist()
for word in cleaned_reviews:
    fd[word] += 1
fd.most_common(5)

cuisines = newdata[['cuisines', 'rating']]
cuisines['cuisines'] = cuisines['cuisines'].astype(str)
cuisines['cuisines'] = cuisines['cuisines'].apply(lambda x: x.lower())
cuisine_tokens = cuisines['cuisines'].apply(tokenizer.tokenize)
all_cuisines = cuisine_tokens.astype(str).str.cat()
cleaned_cuisines = tokenizer.tokenize(all_cuisines)
fd_cuisine = FreqDist()
for cuisine in cleaned_cuisines:
    fd_cuisine[cuisine] += 1

newdata[['reviews_list', 'menu_item', 'dish_liked', 'cuisines']] = newdata[['reviews_list', 'menu_item', 'dish_liked', 'cuisines']].astype('str')
newdata['text'] = newdata['reviews_list'] + ' ' + newdata['menu_item'] + ' ' + newdata['dish_liked'] + ' ' + newdata['cuisines']
text_data = newdata[['text', 'rating']]
text_data['text'] = text_data['text'].apply(lambda x: x.lower())
tokens = text_data['text'].apply(tokenizer.tokenize)
tokens = tokens.apply(lambda x: [token for token in x if token not in stop])

lmtzr = WordNetLemmatizer()
def lem(text):
    return [lmtzr.lemmatize(word) for word in text]
tokens_new = tokens.apply(lem)

le = LabelEncoder()
target = le.fit_transform(text_data['rating'])

X_train, X_test, y_train, y_test = train_test_split(tokens_new, target, test_size=0.3, random_state=0, stratify=target)
t = Tokenizer()
t.fit_on_texts(X_train)
vocab_size = len(t.word_index) + 1
print(vocab_size)