text_plain_output_1.png
from nltk.tokenize import TweetTokenizer

from time import time

import pandas as pd

dataset_columns = ['sentiment', 'ids', 'date', 'flag', 'user', 'text']
dataset_encode = 'ISO-8859-1'
tweet_df = pd.read_csv('../input/twitter-sentiment/training.1600000.processed.noemoticon.csv', encoding=dataset_encode, names=dataset_columns)

new_df = tweet_df[['sentiment', 'text']]

df_pos = tweet_df[tweet_df['sentiment'] == 4]
df_neg = tweet_df[tweet_df['sentiment'] == 0]

new_df = pd.concat([df_pos, df_neg])
len(new_df)

start_time = time()
token = TweetTokenizer(reduce_len=True)
data = []
X = new_df['text'].tolist()
Y = new_df['sentiment'].tolist()
X = new_df['text'].tolist()
Y = new_df['sentiment'].tolist()
for x, y in zip(X, Y):
    if y == 4:
        data.append((token.tokenize(x), 1))
    else:
        data.append((token.tokenize(x), 0))
print('CPU Time:', time() - start_time)
data[:5]