image_output_1.png
from nltk.stem import WordNetLemmatizer

from wordcloud import WordCloud, STOPWORDS

import matplotlib.pyplot as plt

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import re

df1 = pd.read_csv('../input/email-spam-dataset/completeSpamAssassin.csv')
df1 = df1[['Body', 'Label']]
df2 = pd.read_csv('../input/email-spam-dataset/enronSpamSubset.csv')
df2 = df2[['Body', 'Label']]
df3 = pd.read_csv('../input/email-spam-dataset/lingSpam.csv')
df3 = df3[['Body', 'Label']]
data = df1.append(df2).append(df3)
data = data.reset_index(drop=True)

data.dtypes

data["len"] = data["Body"].apply(lambda x: len(str(x)))
data
#------------------------------------------------------------------
print("Quantiles of data : ")
print(data["len"].quantile([0,0.25,0.5,0.75,0.9,0.95,0.99,1]),"\n")

#------------------------------------------------------------------
fig = plt.figure(figsize =(10, 7))
plt.ylim(-50,5000)
#plt.hist(data["len"])
plt.boxplot(data["len"])

import re
data['extracted_body'] = data['Body'].apply(lambda x: re.sub('http\\S+', '', str(x)))

import re
data['extracted_body'] = data['extracted_body'].apply(lambda x: re.sub('\\W+', ' ', x))

count_ham = len(data) - data['Label'].sum()
count_spam = data['Label'].sum()

subset = data.copy()
def word_cloud_plot(subset):
    comment_words = ''
    stopwords = set(STOPWORDS)
    for val in data['extracted_body']:
        val = str(val)
        tokens = val.split()
        for i in range(len(tokens)):
            tokens[i] = tokens[i].lower()
        comment_words += ' '.join(tokens) + ' '
    wordcloud = WordCloud(width=800, height=800, background_color='white', stopwords=stopwords, min_font_size=6).generate(comment_words)
    plt.axis('off')
    plt.tight_layout(pad=0)

data['extracted_body_tk'] = data['extracted_body'].apply(lambda x: x.split())

from nltk.stem import WordNetLemmatizer
stopwords = list(set(STOPWORDS))
lemmatizer = WordNetLemmatizer()
data['extracted_body_tk_lm'] = data['extracted_body_tk'].apply(lambda x: ' '.join([lemmatizer.lemmatize(elem) for elem in x if elem not in stopwords]))

data[['extracted_body_tk', 'extracted_body_tk_lm']][10:11]