text_plain_output_1.png image_output_1.png
from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from sklearn.model_selection import train_test_split

from wordcloud import WordCloud

import emoji

import matplotlib.pyplot as plt

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re # regular expression operations

import wordninja

tweets = pd.read_csv('/kaggle/input/twitter-sentiment-analysis-10/Sentiment140.tenPercent.sample.tweets.tsv', delimiter='\t')

tweets.isnull().values.any().sum()

tweets = tweets.drop_duplicates()

stop_words = stopwords.words('english')
abbreviations = {'a.m.': 'before midday', 'acct': 'account', 'afaik': 'as far as i know', 'afk': 'away from keyboard', 'app': 'application', 'approx': 'approximately', 'apps': 'applications', 'asap': 'as soon as possible', 'asl': 'age, sex, location', 'ave.': 'avenue', 'b.c': 'before christ', 'b2b': 'business to business', 'b4': 'before', 'bae': 'before anyone else', 'bak': 'back at keyboard', 'bbl': 'be back later', 'bbs': 'be back soon', 'be4': 'before', 'blvd': 'boulevard', 'bout': 'about', 'brb': 'be right back', 'btw': 'by the way', 'c/o': 'care of', 'cf': 'compare', 'cia': 'central intelligence agency', 'cu': 'see you', 'cya': 'see you', 'dae': 'does anyone else', 'diy': 'do it yourself', 'dm': 'direct message', 'eet': 'eastern european time', 'eg': 'example', 'encl.': 'enclosed', 'etc': 'and so on', 'fb': 'facebook', 'fc': 'fingers crossed', 'fig': 'figure', 'fwiw': 'for what it is worth', 'fyi': 'for your information', 'ft.': 'feet', 'ft': 'featuring', 'ftl': 'for the loss', 'ftw': 'for the win', 'gal': 'get a life', 'gcse': 'general certificate of secondary education', 'gg': 'good game', 'gl': 'good luck', 'gmt': 'greenwich mean time', 'gn': 'good night', 'goat': 'greatest of all time', 'gps': 'global positioning system', 'gr8': 'great', 'gratz': 'congratulations', 'gyal': 'girl', 'irl': 'in real life', 'hp': 'horsepower', 'hr': 'hour', 'ht': 'height', 'ic': 'i see', 'idc': 'i do not care', 'idgaf': 'i do not give a fuck', 'idk': 'i do not know', 'ie': 'that is', 'iirc': 'if i remember correctly', 'ilu': 'i love you', 'ily': 'i love you', 'imho': 'in my humble opinion', 'imo': 'in my opinion', 'iow': 'in other words', 'jk': 'just kidding', 'l8r': 'later', 'lb': 'pound', 'lbs': 'pounds', 'lmao': 'laugh my ass off', 'lmfao': 'laugh my fucking ass off', 'lol': 'laughing out loud', 'ltd': 'limited', 'm8': 'mate', 'mf': 'motherfucker', 'mofo': 'motherfucker', 'mph': 'miles per hour', 'mr': 'mister', 'ms': 'miss', 'mte': 'my thoughts exactly', 'nbc': 'national broadcasting company', 'nfs': 'not for sale', 'ngl': 'not going to lie', 'nhs': 'national health service', 'nsfw': 'not safe for work', 'rt': 'retweet', 'nth': 'nice to have', 'nvr': 'never', 'nyc': 'new york city', 'oc': 'original content', 'og': 'original', 'oic': 'oh i see', 'omg': 'oh my god', 'omw': 'on my way', 'p.m': 'after midday', 'poc': 'people of color', 'pov': 'point of view', 'pp': 'pages', 'ppl': 'people', 'ps': 'postscript', 'pt': 'point', 'pto': 'please turn over', 'ratchet': 'rude', 'roflol': 'rolling on the floor laughing out loud', 'sk8': 'skate', 'smh': 'shake my head', 'sq': 'square', 'tbh': 'to be honest', 'tbs': 'tablespooful', 'thks': 'thank you', 'tho': 'though', 'thx': 'thank you', 'tia': 'thanks in advance', 'til': 'today i learned', 'tl;dr': 'too long i did not read', 'u2': 'you too', 'w/': 'with', 'tmb': 'tweet me back', 'ttyl': 'talk to you later', 'u': 'you', 'w/o': 'without', 'w8': 'wait', 'wassup': 'what is up', 'wb': 'welcome back', 'wtf': 'what the fuck', 'wtg': 'way to go', 'some1': 'someone', 'yrs': 'years', 'hrs': 'hours', '2morow': 'tomorrow', '2moro': 'tomorrow', '2day': 'today', '4got': 'forget', 'bday': 'birthday', 'b-day': 'birthday'}
def preprocess(text, wordLemm):
    processedText = []
    for tweet in text:
        tweet = tweet.lower()
        tweet = re.sub('((http://)[^ ]*|(https://)[^ ]*|( www\\.)[^ ]*)', 'url', tweet)
        tweet = emoji.demojize(tweet)
        tweet = re.sub('@[^\\s]+', 'user', tweet)
        splitted = wordninja.split(tweet)
        tweet = ' '.join((word for word in splitted))
        for s in abbreviations.keys():
            tweet = tweet.replace(' ' + s, ' ' + abbreviations[s])
        tweet = re.sub('[^a-z\\s]', '', tweet)
        tweet = re.sub('(.)\\1\\1+', '\\1', tweet)
        tweetwords = ''
        for word in tweet.split():
            if len(word) > 1:
                word = wordLemm.lemmatize(word)
                tweetwords += word + ' '
        processedText.append(tweetwords)
    return processedText

dataset = tweets.copy()
wordLemm = WordNetLemmatizer()
dataset['processed_text'] = preprocess(list(tweets['tweet_text']), wordLemm)

wc = WordCloud(max_words=2000, width=1600, height=800).generate(' '.join(dataset[dataset.sentiment_label == 0].processed_text))

wc = WordCloud(max_words=2000, width=1600, height=800).generate(' '.join(dataset[dataset.sentiment_label == 1].processed_text))

X = dataset['processed_text']
y = tweets['sentiment_label']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=6)
print('Train Data size: ', len(X_train))
print('Test Data size: ', len(X_test))