text_html_output_1.png
from nltk.corpus import stopwords

import nltk

import re

import string

def clean_text(text):
    text = text.lower().strip()
    text = ' '.join([w for w in text.split() if len(w) > 2])
    text = re.sub('\\[.*?\\]', '', text)
    text = re.sub('https?://\\S+|www\\.\\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\\w*\\d\\w*', '', text)
    return text

tokenizer = nltk.tokenize.RegexpTokenizer('\\w+')

train['comment'] = train['comment'].apply(tokenizer.tokenize)

def remove_stopwords(word_tokens):
    stop_words = set(stopwords.words('english'))
    filtered_sentence = [w for w in word_tokens if not w in stop_words]
    filtered_sentence = []
    for w in word_tokens:
        if w not in stop_words:
            filtered_sentence.append(w)
    return filtered_sentence

train_word = train.explode('comment')

word_all_rate = train_word.comment.value_counts(ascending=True)

word_all_rate = word_all_rate[word_all_rate > 10]

word_rate_1 = train_word.loc[(train_word['rating'] >= 0) & (train_word['rating'] <= 1.999)]
word_rate_1 = word_rate_1.comment.value_counts(ascending=True)
word_rate_1 = word_rate_1[word_rate_1 > 10]
word_rate_2 = train_word.loc[(train_word['rating'] >= 2) & (train_word['rating'] <= 2.999)]
word_rate_2 = word_rate_2.comment.value_counts(ascending=True)
word_rate_2 = word_rate_2[word_rate_2 > 10]
word_rate_3 = train_word.loc[(train_word['rating'] >= 3) & (train_word['rating'] <= 3.999)]
word_rate_3 = word_rate_3.comment.value_counts(ascending=True)
word_rate_3 = word_rate_3[word_rate_3 > 10]
word_rate_4 = train_word.loc[(train_word['rating'] >= 4) & (train_word['rating'] <= 4.999)]
word_rate_4 = word_rate_4.comment.value_counts(ascending=True)
word_rate_4 = word_rate_4[word_rate_4 > 10]
word_rate_5 = train_word.loc[(train_word['rating'] >= 5) & (train_word['rating'] <= 5.999)]
word_rate_5 = word_rate_5.comment.value_counts(ascending=True)
word_rate_5 = word_rate_5[word_rate_5 > 10]
word_rate_6 = train_word.loc[(train_word['rating'] >= 6) & (train_word['rating'] <= 6.999)]
word_rate_6 = word_rate_6.comment.value_counts(ascending=True)
word_rate_6 = word_rate_6[word_rate_6 > 10]
word_rate_7 = train_word.loc[(train_word['rating'] >= 7) & (train_word['rating'] <= 7.999)]
word_rate_7 = word_rate_7.comment.value_counts(ascending=True)
word_rate_7 = word_rate_7[word_rate_7 > 10]
word_rate_8 = train_word.loc[(train_word['rating'] >= 8) & (train_word['rating'] <= 8.999)]
word_rate_8 = word_rate_8.comment.value_counts(ascending=True)
word_rate_8 = word_rate_8[word_rate_8 > 10]
word_rate_9 = train_word.loc[(train_word['rating'] >= 9) & (train_word['rating'] <= 9.999)]
word_rate_9 = word_rate_9.comment.value_counts(ascending=True)
word_rate_9 = word_rate_9[word_rate_9 > 10]
word_rate_10 = train_word.loc[train_word['rating'] >= 10]
word_rate_10 = word_rate_10.comment.value_counts(ascending=True)
word_rate_10 = word_rate_10[word_rate_10 > 10]

word_rate_list = [word_rate_1, word_rate_2, word_rate_3, word_rate_4, word_rate_5, word_rate_6, word_rate_7, word_rate_8, word_rate_9, word_rate_10]

rate = ['1_', '2_', '3_', '4_', '5_', '6_', '7_', '8_', '9_', '10_', 'predict']

def naive_bayes(text, word_all_rate, word_rate_1, smoothing):
    if smoothing != True:
        if (text in word_all_rate) & (text in word_rate_1):
            return word_rate_1[text] / word_rate_1.size
        else:
            return 0
    elif (text in word_all_rate) & (text in word_rate_1):
        return (word_rate_1[text] + 1) / (word_rate_1.size + 10)
    else:
        return 1 / (word_rate_1.size + 10)

def predict_rate(df, word_all_rate, word_rate_list, rate):
    df['comment'] = df['comment'].apply(tokenizer.tokenize)
    df['comment'] = df['comment'].apply(remove_stopwords)
    exploded = df.explode('comment')
    for i in range(10):
        exploded[i + 1] = exploded['comment'].apply(lambda x: naive_bayes(x, word_all_rate, word_rate_list[i - 1], 1))
    for i in df.index:
        ff = exploded.loc[exploded.index == i].prod()
        max_ = -1
        position = 0
        for j, k in zip(range(10), rate):
            df.loc[df.index == i, k] = ff[j + 1]
            if max_ < ff[j + 1]:
                max_ = ff[j + 1]
                position = j + 1
        df.loc[df.index == i, 'predict'] = position
    return df

test_reduce_num = test[:500]

test_reduce_num = predict_rate(test_reduce_num, word_all_rate, word_rate_list, rate)

test_reduce_num