text_plain_output_1.png
from nltk.corpus import stopwords

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import string

df = pd.read_csv('/kaggle/input/sentiment140/training.1600000.processed.noemoticon.csv', encoding='latin', header=None)
df.columns = ['sentiment', 'id', 'date', 'query', 'user_id', 'text']
df = df.drop(['id', 'date', 'query', 'user_id'], axis=1)

df.sentiment = df.sentiment.map({0: 0, 4: 1})

stop_words = set(stopwords.words('english'))
def data_preprocessing(text):
    text = text.lower()
    text = re.sub('<.*?>', '', text)
    text = ''.join([c for c in text if c not in string.punctuation])
    text = [word for word in text.split() if word not in stop_words]
    text = ' '.join(text)
    return text
df['text'] = df['text'].astype(str).apply(data_preprocessing)

MAX_LEN = max([len(x.split()) for x in df['text']])
print(MAX_LEN)