text_html_output_1.png
from collections import Counter

from sklearn.feature_extraction.text import CountVectorizer

import matplotlib.pyplot as plt

import pandas as pd

import re

import seaborn as sns

import string

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn import preprocessing, feature_extraction, model_selection, linear_model
import seaborn as sns
sns.set()

review_text_train = pd.read_csv('../input/review_text_train.csv', index_col=False, delimiter=',', header=0)
review_text_test = pd.read_csv('../input/review_text_test.csv', index_col=False, delimiter=',', header=0)
review_meta_train = pd.read_csv('../input/review_meta_train.csv', index_col=False, delimiter=',', header=0)
review_meta_test = pd.read_csv('../input/review_meta_test.csv', index_col=False, delimiter=',', header=0)

df_train = review_text_train.copy()

df_train.isna().sum()

df_test = review_text_test.copy()
df_test['vote_funny'] = review_meta_test.vote_funny
df_test['vote_cool'] = review_meta_test.vote_cool
df_test['vote_useful'] = review_meta_test.vote_useful
df_test

df_test.isna().sum()

def clean_text_round_1(text):
    text = text.lower()
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\\w*\\d\\w*', '', text)
    return text
round_1 = lambda x: clean_text_round_1(x)

data_review_cleaned = pd.DataFrame(df_train.review.apply(round_1))

def clean_text_round_2(text):
    text = re.sub('\n', ' ', text)
    text = re.sub('[""..._]', '', text)
    return text
round2 = lambda x: clean_text_round_2(x)

data_review_cleaned = pd.DataFrame(data_review_cleaned.review.apply(round2))

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(stop_words='english')
data_cv = cv.fit_transform(data_review_cleaned.review)
data_dtm = pd.DataFrame(data_cv.toarray(), columns=cv.get_feature_names())
data_dtm.index = data_review_cleaned.index
data_dtm

data_dtm_transposed = data_dtm.T

data_review_cleaned_test = pd.DataFrame(df_test.review.apply(round_1))
data_review_cleaned_test = pd.DataFrame(data_review_cleaned_test.review.apply(round2))

top_words = {}
for o in data_dtm_transposed.columns:
    top = data_dtm_transposed[o].sort_values(ascending=False).head(50)
    top_words[o] = list(zip(top.index, top.values))

data = data_dtm_transposed

from collections import Counter
words = []
for user_id in data.columns:
    top = [word for word, count in top_words[user_id]]
    for t in top:
        words.append(t)

common_words_count = Counter(words).most_common()

df_common_words = pd.DataFrame(common_words_count, columns=['word', 'count'])
df_common_words

fig, ax = plt.subplots()
fig.set_size_inches(15, 12)
ax = sns.barplot(x='count', y='word', data=df_common_words[:30])
ax.set_title('Top 30 Words in the Corpus', size=24)
ax.set_xlabel('Count', size=20)
ax.set_ylabel('Words', size=20)
fig.savefig('top_30_words.png')