text_plain_output_1.png
from nltk.corpus import stopwords

from nltk.stem import SnowballStemmer

from sklearn.model_selection import train_test_split

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

path = '/kaggle/input/sentiment140/training.1600000.processed.noemoticon.csv'
df = pd.read_csv(path, header=None)

df.columns = ['target', 'ids', 'date', 'flag', 'user', 'text']

df.sample(10)

df.shape

stop_words = stopwords.words('english')
stemmer = SnowballStemmer('english')
cleaning = '@\\S+|https?:\\S+|http?:\\S|[^A-Za-z0-9]+'
def preprocess(text, stem=False):
    text = re.sub(cleaning, ' ', str(text).lower()).strip()
    tokens = []
    for token in text.split():
        if token not in stop_words:
            if stem:
                tokens.append(stemmer.stem(token))
            else:
                tokens.append(token)
    return ' '.join(tokens)

df.text = df.text.apply(lambda x: preprocess(x, stem=True))

x = df.text
y = df.target
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=42)

df