text_plain_output_1.png
from nltk.corpus import stopwords

from nltk.stem.porter import PorterStemmer

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')

stop = stopwords.words('english')
stemmer = PorterStemmer()
def clean_text(text):
    processed_text = re.sub('(@\\[A-Za-z0-9]+)|([^A-Za-z \\t])|(\\w+:\\/\\/\\S+)|^rt|http.+?', '', text)
    processed_text = processed_text.lower()
    processed_text = [word for word in processed_text.split() if word not in stop]
    processed_text = ' '.join([stemmer.stem(word) for word in processed_text])
    return processed_text
def preprocessing(df):
    df.keyword.fillna('', inplace=True)
    df.location.fillna('', inplace=True)
    df.text = df.keyword + df.location + df.text
    df.text = df.text.apply(lambda text: clean_text(text))
    df.drop(columns=['keyword', 'location'], inplace=True)
    return df

train_df = preprocessing(train_df)
test_df = preprocessing(test_df)

train_df.head()