text_html_output_1.png
from collections import Counter

from sklearn.feature_extraction.text import CountVectorizer

import pandas as pd

import re

import string

review_text_train = pd.read_csv('../input/review_text_train.csv', index_col=False, delimiter=',', header=0)
review_text_test = pd.read_csv('../input/review_text_test.csv', index_col=False, delimiter=',', header=0)
review_meta_train = pd.read_csv('../input/review_meta_train.csv', index_col=False, delimiter=',', header=0)
review_meta_test = pd.read_csv('../input/review_meta_test.csv', index_col=False, delimiter=',', header=0)

df_train = review_text_train.copy()

df_train.isna().sum()

df_test = review_text_test.copy()
df_test['vote_funny'] = review_meta_test.vote_funny
df_test['vote_cool'] = review_meta_test.vote_cool
df_test['vote_useful'] = review_meta_test.vote_useful
df_test

df_test.isna().sum()

def clean_text_round_1(text):
    text = text.lower()
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\\w*\\d\\w*', '', text)
    return text
round_1 = lambda x: clean_text_round_1(x)

data_review_cleaned = pd.DataFrame(df_train.review.apply(round_1))

def clean_text_round_2(text):
    text = re.sub('\n', ' ', text)
    text = re.sub('[""..._]', '', text)
    return text
round2 = lambda x: clean_text_round_2(x)

data_review_cleaned = pd.DataFrame(data_review_cleaned.review.apply(round2))

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(stop_words='english')
data_cv = cv.fit_transform(data_review_cleaned.review)
data_dtm = pd.DataFrame(data_cv.toarray(), columns=cv.get_feature_names())
data_dtm.index = data_review_cleaned.index
data_dtm

data_dtm_transposed = data_dtm.T

data_review_cleaned_test = pd.DataFrame(df_test.review.apply(round_1))
data_review_cleaned_test = pd.DataFrame(data_review_cleaned_test.review.apply(round2))

top_words = {}
for o in data_dtm_transposed.columns:
    top = data_dtm_transposed[o].sort_values(ascending=False).head(50)
    top_words[o] = list(zip(top.index, top.values))

data = data_dtm_transposed

from collections import Counter
words = []
for user_id in data.columns:
    top = [word for word, count in top_words[user_id]]
    for t in top:
        words.append(t)

common_words_count = Counter(words).most_common()

df_common_words = pd.DataFrame(common_words_count, columns=['word', 'count'])
df_common_words