text_html_output_1.png
from nltk.corpus import stopwords

import pandas as pd

import re

train_dataset = pd.read_csv('../input/nlp-getting-started/train.csv')
test_dataset = pd.read_csv('../input/nlp-getting-started/test.csv')

train_dataset.shape

train_dataset.fillna('', inplace=True)
test_dataset.fillna('', inplace=True)

dataset = pd.DataFrame()
test_dataset_cleaned = pd.DataFrame()
dataset['all_combined'] = train_dataset['keyword'] + ' ' + train_dataset['location'] + ' ' + train_dataset['text']
test_dataset_cleaned['all_combined'] = test_dataset['keyword'] + ' ' + test_dataset['location'] + ' ' + test_dataset['text']
test_dataset_cleaned.shape

def clean(data):
    data = data.lower()
    data = re.sub('https?://\\S+|www\\.\\S+', ' ', data)
    data = re.sub('\\W', ' ', data)
    data = re.sub('\n', ' ', data)
    data = re.sub(' +', ' ', data)
    data = re.sub('^ ', ' ', data)
    data = re.sub(' $', ' ', data)
    data = re.sub('#', ' ', data)
    data = re.sub('@', ' ', data)
    data = re.sub('[^a-zA-Z]', ' ', data)
    return data

stop = set(stopwords.words('english'))
def remove_stopwords(data):
    words = [word for word in data if word not in stop]
    words = ''.join(words).split()
    words = [words.lower() for words in data.split()]
    return words

dataset['all_cleaned'].apply(remove_stopwords)
test_dataset_cleaned['all_cleaned'].apply(remove_stopwords)