text_plain_output_2.png text_plain_output_1.png
from nltk.classify.scikitlearn import SklearnClassifier

from nltk.corpus import stopwords

from nltk.stem.wordnet import WordNetLemmatizer

from nltk.tag import pos_tag

from nltk.tokenize import TweetTokenizer

from sklearn.naive_bayes import MultinomialNB,BernoulliNB

from time import time

import nltk

import pandas as pd

import random

dataset_columns = ['sentiment', 'ids', 'date', 'flag', 'user', 'text']
dataset_encode = 'ISO-8859-1'
tweet_df = pd.read_csv('../input/twitter-sentiment/training.1600000.processed.noemoticon.csv', encoding=dataset_encode, names=dataset_columns)

new_df = tweet_df[['sentiment', 'text']]

df_pos = tweet_df[tweet_df['sentiment'] == 4]
df_neg = tweet_df[tweet_df['sentiment'] == 0]

new_df = pd.concat([df_pos, df_neg])
len(new_df)

start_time = time()
token = TweetTokenizer(reduce_len=True)
data = []
X = new_df['text'].tolist()
Y = new_df['sentiment'].tolist()
X = new_df['text'].tolist()
Y = new_df['sentiment'].tolist()
for x, y in zip(X, Y):
    if y == 4:
        data.append((token.tokenize(x), 1))
    else:
        data.append((token.tokenize(x), 0))
data[:5]

import nltk
nltk.download('all')

def lemmatize_sentence(tokens):
    lemmatizer = WordNetLemmatizer()
    lemmatized_sentence = []
    for word, tag in pos_tag(tokens):
        if tag.startswith('NN'):
            pos = 'n'
        elif tag.startswith('VB'):
            pos = 'v'
        else:
            pos = 'a'
        lemmatized_sentence.append(lemmatizer.lemmatize(word, pos))
    return lemmatized_sentence

import re, string
from nltk.corpus import stopwords
STOP_WORDS = stopwords.words('english')
def cleaned(token):
    if token == 'u':
        return 'you'
    if token == 'r':
        return 'are'
    if token == 'some1':
        return 'someone'
    if token == 'yrs':
        return 'years'
    if token == 'hrs':
        return 'hours'
    if token == 'mins':
        return 'minutes'
    if token == 'secs':
        return 'seconds'
    if token == 'pls' or token == 'plz':
        return 'please'
    if token == '2morow':
        return 'tomorrow'
    if token == '2day':
        return 'today'
    if token == '4got' or token == '4gotten':
        return 'forget'
    if token == 'amp' or token == 'quot' or token == 'lt' or (token == 'gt') or (token == '½25'):
        return ''
    return token

def remove_noise(tweet_tokens):
    cleaned_tokens = []
    for token, tag in pos_tag(tweet_tokens):
        token = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '', token)
        token = re.sub('(@[A-Za-z0-9_]+)', '', token)
        if tag.startswith('NN'):
            pos = 'n'
        elif tag.startswith('VB'):
            pos = 'v'
        else:
            pos = 'a'
        lemmatizer = WordNetLemmatizer()
        token = lemmatizer.lemmatize(token, pos)
        cleaned_token = cleaned(token.lower())
        if cleaned_token not in string.punctuation and len(cleaned_token) > 2 and (cleaned_token not in STOP_WORDS):
            cleaned_tokens.append(cleaned_token)
    return cleaned_tokens

start_time = time()
def list_to_dict(cleaned_tokens):
    return dict(([token, True] for token in cleaned_tokens))
cleaned_tokens_list = []
for tokens, label in data:
    cleaned_tokens_list.append((remove_noise(tokens), label))
start_time = time()
final_data = []
for tokens, label in cleaned_tokens_list:
    final_data.append((list_to_dict(tokens), label))
final_data[:5]

import random
random.Random(140).shuffle(final_data)
trim_index = int(len(final_data) * 0.9)
train_data = final_data[:trim_index]
test_data = final_data[trim_index:]

start_time = time()
from nltk import classify
from nltk import NaiveBayesClassifier
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
classify_BNB = SklearnClassifier(BernoulliNB())
classifier = classify_BNB.train(train_data)
print('Accuracy on train data:', nltk.classify.accuracy(classifier, train_data) * 100)
print('Accuracy on test data:', nltk.classify.accuracy(classifier, test_data) * 100)